diff --git a/Cargo.lock b/Cargo.lock index e8bfa39..165ff9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -265,7 +265,7 @@ dependencies = [ [[package]] name = "text_summarize" -version = "0.1.0" +version = "0.2.0" dependencies = [ "chain-module-sdk", "serde", diff --git a/MODULE.de.md b/MODULE.de.md index 8a73e43..df3421d 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -17,7 +17,7 @@ audit-taugliche Modell-Herkunfts-Felder. | `style` | text | Optionaler Stil-Hinweis (z.B. `one paragraph`, `three bullet points`, `a tweet`). Default: `one paragraph`. | | `language` | text | Optionaler Ziel-Sprache (z.B. `German`, `ja-JP`). Leer = Quellsprache. | | `endpoint` | text | Chat-Endpunkt passend zum gewählten `api` (Ollama `/api/chat`, OpenAI-kompatibel `/v1/chat/completions` — vLLM u. a., Anthropic `/v1/messages`). | -| `api` | text | Optionales Wire-Format: `ollama` (Default), `openai`, `anthropic`. | +| `api` | text | Optionales Wire-Format: `ollama` (Default), `vllm`, `openai`, `anthropic` (`vllm` = OpenAI-Wire mit vLLM-spezifischen Hinweisen). | | `model` | text | Modell-ID am Endpunkt. | | `api_key` | text | Optionaler Bearer-Token für Cloud-Endpunkte. | diff --git a/MODULE.md b/MODULE.md index 6468e9b..cd2a71e 100644 --- a/MODULE.md +++ b/MODULE.md @@ -17,7 +17,7 @@ model-provenance fields. | `style` | text | Optional style hint (e.g. `one paragraph`, `three bullet points`, `a tweet`). Default: `one paragraph`. | | `language` | text | Optional output-language hint (e.g. `German`, `ja-JP`). Empty = same as source. | | `endpoint` | text | Chat endpoint URL matching the selected `api` (Ollama `/api/chat`, OpenAI-compatible `/v1/chat/completions` — vLLM etc., Anthropic `/v1/messages`). | -| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | +| `api` | text | Optional wire format: `ollama` (default), `vllm`, `openai`, `anthropic` (`vllm` = the OpenAI wire with vLLM-specific hints). | | `model` | text | Model identifier the endpoint serves. | | `api_key` | text | Optional bearer token for cloud-hosted endpoints. | diff --git a/module.yaml b/module.yaml index 48fae31..ff6444a 100644 --- a/module.yaml +++ b/module.yaml @@ -52,13 +52,20 @@ inputs: type: text description: en: | - Optional wire format: "ollama" (default), "openai" - (OpenAI-compatible servers such as vLLM or LM Studio), - or "anthropic" (Messages API). Empty = ollama. + Optional wire format: "ollama" (default), "vllm" + (self-hosted vLLM), "openai" (OpenAI or other + OpenAI-compatible servers such as LM Studio), or + "anthropic" (Messages API). vllm and openai speak the + same wire; the separate value exists so you can write + what you mean and get vLLM-specific hints. Empty = ollama. de: | - Optionales Wire-Format: "ollama" (Default), "openai" - (OpenAI-kompatible Server wie vLLM oder LM Studio) - oder "anthropic" (Messages API). Leer = ollama. + Optionales Wire-Format: "ollama" (Default), "vllm" + (selbst gehostetes vLLM), "openai" (OpenAI oder andere + OpenAI-kompatible Server wie LM Studio) oder "anthropic" + (Messages API). vllm und openai sprechen dieselbe Wire; + der eigene Wert existiert, damit Du schreibst, was Du + meinst, und vLLM-spezifische Hinweise bekommst. + Leer = ollama. model: type: text description: diff --git a/src/lib.rs b/src/lib.rs index b396ec3..5192891 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,9 @@ //! `llm.chat`. //! //! The wire format is selected by the optional `api` input: -//! `ollama` (default, unchanged v0.1.x behavior), `openai` -//! (OpenAI-compatible `/v1/chat/completions` — vLLM, LM Studio, -//! LiteLLM, cloud OpenAI), or `anthropic` (Messages API). The +//! `ollama` (default, unchanged v0.1.x behavior), `vllm` +//! (self-hosted vLLM), `openai` (any other OpenAI-compatible +//! server), or `anthropic` (Messages API). The //! client logic in `llm.rs` is kept in lockstep with `llm.chat`. mod llm; @@ -85,7 +85,12 @@ fn llm_error_to_module_error( )), (Api::Openai, LlmError::Http(detail)) => ModuleError::internal(format!( "LLM endpoint {endpoint} not reachable ({detail}). Expected an \ - OpenAI-compatible server (OpenAI, vLLM, ...) at a /v1/chat/completions URL." + OpenAI-compatible server at a /v1/chat/completions URL." + )), + (Api::Vllm, LlmError::Http(detail)) => ModuleError::internal(format!( + "LLM endpoint {endpoint} not reachable ({detail}). Is vLLM running? \ + Start it with `vllm serve {model}`, then point `endpoint` at \ + http://:8000/v1/chat/completions." )), (Api::Anthropic, LlmError::Http(detail)) => ModuleError::internal(format!( "LLM endpoint {endpoint} not reachable ({detail}). Expected the \ @@ -109,7 +114,7 @@ fn llm_error_to_module_error( ModuleError::invalid_input(format!("missing required input '{name}'")) } (_, LlmError::UnsupportedApi(raw)) => ModuleError::invalid_input(format!( - "unsupported api '{raw}' (expected: ollama, openai, anthropic)" + "unsupported api '{raw}' (expected: ollama, openai, vllm, anthropic)" )), } } diff --git a/src/llm.rs b/src/llm.rs index 90565f5..12a890c 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -33,7 +33,7 @@ const ANTHROPIC_VERSION: &str = "2023-06-01"; pub enum LlmError { #[error("missing required input '{0}'")] MissingInput(&'static str), - #[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")] + #[error("unsupported api '{0}' (expected: ollama, openai, vllm, anthropic)")] UnsupportedApi(String), #[error("http error: {0}")] Http(String), @@ -48,6 +48,13 @@ pub enum LlmError { pub enum Api { Ollama, Openai, + /// vLLM speaks the OpenAI wire format verbatim; it is a + /// separate variant ONLY so operators can write what they + /// mean (`api: vllm`) and get vLLM-specific error hints — + /// "openai" would read as the cloud company, not a + /// self-hosted server. Same rationale as the hub's + /// SystemLlmProvider vocabulary. + Vllm, Anthropic, } @@ -58,6 +65,7 @@ impl Api { match raw.trim().to_ascii_lowercase().as_str() { "" | "ollama" => Ok(Api::Ollama), "openai" => Ok(Api::Openai), + "vllm" => Ok(Api::Vllm), "anthropic" => Ok(Api::Anthropic), other => Err(LlmError::UnsupportedApi(other.to_string())), } @@ -92,7 +100,7 @@ pub struct ChatParams<'a> { pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> { let mut headers = Vec::with_capacity(2); match p.api { - Api::Ollama | Api::Openai => { + Api::Ollama | Api::Openai | Api::Vllm => { if !p.api_key.is_empty() { headers.push(("Authorization", format!("Bearer {}", p.api_key))); } @@ -261,19 +269,19 @@ pub fn chat_with_identity( } let body = match p.api { Api::Ollama => build_ollama_body(p), - Api::Openai => build_openai_body(p), + Api::Openai | Api::Vllm => build_openai_body(p), Api::Anthropic => build_anthropic_body(p), }; let headers = build_headers(p); let response_body = client.post_json(p.endpoint, &body, &headers)?; let response = match p.api { Api::Ollama => extract_ollama_content(&response_body)?, - Api::Openai => extract_openai_content(&response_body)?, + Api::Openai | Api::Vllm => extract_openai_content(&response_body)?, Api::Anthropic => extract_anthropic_content(&response_body)?, }; let model_digest = match p.api { Api::Ollama => probe_model_digest(client, p), - Api::Openai | Api::Anthropic => None, + Api::Openai | Api::Vllm | Api::Anthropic => None, }; Ok(ChatWithIdentity { response, @@ -717,4 +725,26 @@ mod tests { Err(LlmError::MissingInput("prompt")) )); } + + /// Guard: `api: vllm` exists ONLY for operator clarity — the + /// request it emits (URL, body, headers) must stay identical to + /// `api: openai`. If the two ever diverge, that is a new wire + /// dialect and needs its own deliberate design, not drift. + #[test] + fn vllm_is_the_openai_wire_verbatim() { + assert_eq!(Api::parse("vllm").unwrap(), Api::Vllm); + let openai_reply = r#"{"choices":[{"message":{"role":"assistant","content":"x"}}]}"#; + let recorded: Vec = [Api::Vllm, Api::Openai] + .into_iter() + .map(|api| { + let client = MockClient::new(vec![Ok(openai_reply.to_string())]); + chat_with_identity(&client, ¶ms(api)).unwrap(); + client.calls.into_inner().pop().unwrap() + }) + .collect(); + assert_eq!( + recorded[0], recorded[1], + "vllm request drifted from openai" + ); + } }