diff --git a/Cargo.lock b/Cargo.lock index e2d8ab4..c510695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.3.1" +version = "0.2.1" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index caceacd..6391d14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.3.1" +version = "0.2.1" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.de.md b/MODULE.de.md index 8e1bb5b..21c2d00 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -10,7 +10,7 @@ die Anthropic Messages API. ## Capability -- `llm.chat@0.3.0` +- `llm.chat@0.2.0` ## Eingaben @@ -40,8 +40,6 @@ die Anthropic Messages API. | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | | `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | -| `tokens_prompt` | text | Was die Anfrage gekostet hat, nach der Zählung des Backends. Leer, wenn die Antwort keine Zählung enthielt. | -| `tokens_completion` | text | Was die Antwort gekostet hat, gleiche Regel. Ollama zählt in `eval_count`, das OpenAI-Format in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Zusammen beantworten die drei `model_*`-Ausgaben die Audit- Frage: „Welches genaue Modell hat diese Antwort erzeugt?" — diff --git a/MODULE.md b/MODULE.md index f697dd1..29a511e 100644 --- a/MODULE.md +++ b/MODULE.md @@ -10,7 +10,7 @@ Anthropic Messages API. ## Capability -- `llm.chat@0.3.0` +- `llm.chat@0.2.0` ## Inputs @@ -21,7 +21,6 @@ Anthropic Messages API. | `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | | `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | | `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. | -| `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with ``. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | | `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | @@ -41,8 +40,6 @@ Anthropic Messages API. | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | | `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | -| `tokens_prompt` | text | What the prompt cost, as the backend counted it. Empty when the answer carried no count. | -| `tokens_completion` | text | What the answer cost, same rule. Ollama counts in `eval_count`, the OpenAI shape in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Together the three `model_*` outputs answer the audit question: "which exact model produced this response?" — that diff --git a/module.wasm b/module.wasm index c600c7f..b7e865b 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 62ea8af..43727f8 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.3.1 +version: 0.2.2 # Capability provided by this module. provides: - capability: llm.chat - version: 0.3.1 + version: 0.2.2 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -129,27 +129,6 @@ inputs: großen lokalen Modell). Mit einer anderen api wird der Wert abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt, die die anderen Backends nicht erfüllen können. - think: - type: text - description: - en: | - Whether a reasoning model runs its thinking phase: true or - false; empty leaves the server default. Ollama lets - qwen3-type models think by default, which takes a call from - about 10 to about 50 seconds. Ollama receives think, vLLM - chat_template_kwargs.enable_thinking. The OpenAI wire has no - such switch, so a value is refused there; for Anthropic false - is accepted (extended thinking is off unless requested) and - true is refused. - de: | - Ob ein Reasoning-Modell seine Denkphase durchläuft: true oder - false; leer lässt die Voreinstellung des Servers. Ollama lässt - Modelle wie qwen3 standardmäßig denken, ein Aufruf dauert - dadurch rund 50 statt 10 Sekunden. Ollama bekommt think, vLLM - chat_template_kwargs.enable_thinking. Das OpenAI-Format kennt - keinen solchen Schalter, dort wird ein Wert abgelehnt; bei - Anthropic wird false angenommen (erweitertes Denken ist ohne - Anforderung aus) und true abgelehnt. system_prompt: type: text description: @@ -192,32 +171,6 @@ outputs: keine Digest-API bereit). Zusammen mit model_endpoint und model_name beantwortet das die Audit-Frage "welches genaue Modell hat diese Antwort erzeugt?" - tokens_prompt: - type: text - description: - en: | - How many tokens the prompt cost, as the backend counted it. - Empty when the answer carried no count: a zero would read - like a measurement nobody took. The hub lifts this field - into the audit log automatically. - de: | - Wie viele Token die Anfrage gekostet hat, nach der Zählung - des Backends. Leer, wenn die Antwort keine Zählung enthielt: - eine Null würde wie eine Messung aussehen, die niemand - gemacht hat. Der Hub hebt dieses Feld selbst in den Prüfpfad. - tokens_completion: - type: text - description: - en: | - How many tokens the answer cost, same rules as - tokens_prompt. Ollama counts in `eval_count`, the OpenAI - shape in `usage.completion_tokens`, Anthropic in - `usage.output_tokens`. - de: | - Wie viele Token die Antwort gekostet hat, gleiche Regeln wie - tokens_prompt. Ollama zählt in `eval_count`, das - OpenAI-Format in `usage.completion_tokens`, Anthropic in - `usage.output_tokens`. # Permissions required. # diff --git a/src/lib.rs b/src/lib.rs index d4a432d..4b31f50 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,11 +38,6 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("keep_alive") .and_then(payload_text) .unwrap_or_default(); - // A flow may write `think: false` (JSON boolean) or "false". - let think = match inputs.get("think") { - Some(Payload::Text(s)) | Some(Payload::Json(s)) => s.clone(), - _ => String::new(), - }; let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default(); let temperature = sampling_input("temperature"); let top_p = sampling_input("top_p"); @@ -71,7 +66,6 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { system_prompt: &system_prompt, prompt: &prompt, keep_alive: &keep_alive, - think: &think, sampling: crate::llm::Sampling { temperature: &temperature, top_p: &top_p, @@ -82,20 +76,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let result = crate::llm::chat_with_identity(&client, ¶ms) .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; - // The two token counts are the cost of this call, as the backend - // counted it. They stay empty where the answer carried no count, - // because an invented zero would look like a measurement. - let (tokens_prompt, tokens_completion) = match result.tokens { - Some((prompt, completion)) => (prompt.to_string(), completion.to_string()), - None => (String::new(), String::new()), - }; Ok(Outputs::new() .with_text("response", result.response) .with_text("model_endpoint", endpoint) .with_text("model_name", model) - .with_text("model_digest", result.model_digest.unwrap_or_default()) - .with_text("tokens_prompt", tokens_prompt) - .with_text("tokens_completion", tokens_completion)) + .with_text("model_digest", result.model_digest.unwrap_or_default())) } fn payload_text(p: &Payload) -> Option { diff --git a/src/llm.rs b/src/llm.rs index 8fc1d6b..bb569e4 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -97,11 +97,6 @@ pub struct ChatParams<'a> { /// the server default, which unloads after five minutes and /// makes the next first request pay a 30 to 40 second reload. pub keep_alive: &'a str, - /// Whether a reasoning model runs its thinking phase: "true", - /// "false", or empty for the server default. Ollama lets - /// qwen3-type models think by default, which takes a call from - /// about 10 to about 50 seconds. - pub think: &'a str, /// Sampling controls. Empty means "leave the server default". /// /// `seed` and `temperature: 0` are what make a run repeatable, @@ -209,8 +204,6 @@ struct OllamaRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] keep_alive: Option>, #[serde(skip_serializing_if = "Option::is_none")] - think: Option, - #[serde(skip_serializing_if = "Option::is_none")] options: Option, } @@ -247,27 +240,6 @@ struct OpenAiRequest<'a> { top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] seed: Option, - /// vLLM only: passed to the chat template, where qwen3-type - /// templates read `enable_thinking`. - #[serde(skip_serializing_if = "Option::is_none")] - chat_template_kwargs: Option, -} - -#[derive(Serialize)] -struct ChatTemplateKwargs { - enable_thinking: bool, -} - -/// Parse the `think` input: empty is "server default". -pub fn parse_think(raw: &str) -> Result, LlmError> { - match raw.trim().to_ascii_lowercase().as_str() { - "" => Ok(None), - "true" => Ok(Some(true)), - "false" => Ok(Some(false)), - _ => Err(LlmError::UnsupportedApi(format!( - "think must be true or false, got {raw:?}" - ))), - } } #[derive(Serialize)] @@ -316,7 +288,6 @@ pub fn build_ollama_body(p: &ChatParams) -> Result { messages: build_messages(p), stream: false, keep_alive: parse_keep_alive(p.keep_alive), - think: parse_think(p.think)?, options: (!options.is_empty()).then_some(options), }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) @@ -335,22 +306,6 @@ pub fn build_openai_body(p: &ChatParams) -> Result { .to_string(), )); } - // vLLM hands the switch to the chat template. The OpenAI wire - // has no switch for it; refused rather than dropped, because a - // caller who sets it expects the call to take the time it asked - // for. - let think = parse_think(p.think)?; - let chat_template_kwargs = match (p.api, think) { - (_, None) => None, - (Api::Vllm, Some(enable_thinking)) => Some(ChatTemplateKwargs { enable_thinking }), - (_, Some(_)) => { - return Err(LlmError::UnsupportedApi( - "think has no equivalent on the OpenAI chat wire; use \ - api: ollama or vllm, or drop the input" - .to_string(), - )); - } - }; let req = OpenAiRequest { model: p.model, messages: build_messages(p), @@ -358,7 +313,6 @@ pub fn build_openai_body(p: &ChatParams) -> Result { temperature: parse_sampling("temperature", p.sampling.temperature, false)?, top_p: parse_sampling("top_p", p.sampling.top_p, false)?, seed: parse_sampling("seed", p.sampling.seed, true)?, - chat_template_kwargs, }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } @@ -377,17 +331,6 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result { .to_string(), )); } - // Extended thinking is off unless requested, so "false" is what - // the server does anyway. Turning it on needs a token budget this - // module does not take yet. - if parse_think(p.think)? == Some(true) { - return Err(LlmError::UnsupportedApi( - "think: true is not supported for the Anthropic Messages API \ - here (extended thinking needs a token budget); use false or \ - drop the input" - .to_string(), - )); - } let req = AnthropicRequest { model: p.model, max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -450,41 +393,6 @@ pub fn extract_anthropic_content(body: &str) -> Result { pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, - /// How many tokens the call cost, as the backend counted them: - /// (prompt, completion). `None` where the answer carries no - /// count, which is honest about a number nobody measured. - pub tokens: Option<(u64, u64)>, -} - -/// The token counts of one answer, per wire format. -/// -/// Three dialects, three spellings of the same two numbers: Ollama -/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape -/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic -/// in `usage.input_tokens` and `usage.output_tokens`. A missing -/// count is `None` rather than a zero, because zero is a -/// measurement and this is the absence of one. -pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> { - let v: serde_json::Value = serde_json::from_str(body).ok()?; - let pair = match api { - Api::Ollama => ( - v.get("prompt_eval_count").and_then(|n| n.as_u64()), - v.get("eval_count").and_then(|n| n.as_u64()), - ), - Api::Openai | Api::Vllm => ( - v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()), - v.pointer("/usage/completion_tokens") - .and_then(|n| n.as_u64()), - ), - Api::Anthropic => ( - v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()), - v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()), - ), - }; - match pair { - (None, None) => None, - (prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))), - } } /// Run one chat call in the selected wire format AND, for Ollama, @@ -521,11 +429,9 @@ pub fn chat_with_identity( Api::Ollama => probe_model_digest(client, p), Api::Openai | Api::Vllm | Api::Anthropic => None, }; - let tokens = extract_tokens(p.api, &response_body); Ok(ChatWithIdentity { response, model_digest, - tokens, }) } @@ -621,7 +527,6 @@ mod tests { system_prompt: "", prompt: "hello", keep_alive: "", - think: "", sampling: Sampling::default(), } } @@ -712,7 +617,6 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", - think: "", sampling: Sampling::default(), }; let body = build_openai_body(&p).expect("body builds"); @@ -784,7 +688,6 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", - think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -807,7 +710,6 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", - think: "", sampling: Sampling::default(), }; let body = build_anthropic_body(&p).expect("body builds"); @@ -890,7 +792,6 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", - think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -914,47 +815,6 @@ mod tests { assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); } - /// Three dialects, three spellings of the same two numbers. - #[test] - fn every_dialect_reports_its_token_counts() { - let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#; - assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7))); - - let openai = r#"{"choices":[{"message":{"content":"ja"}}], - "usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#; - assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7))); - assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7))); - - let anthropic = r#"{"content":[{"text":"ja"}], - "usage":{"input_tokens":31,"output_tokens":7}}"#; - assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7))); - } - - /// A backend that counts nothing leaves the fields empty, rather - /// than reporting a zero nobody measured. - #[test] - fn an_answer_without_counts_reports_none() { - assert_eq!( - extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#), - None - ); - assert_eq!( - extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#), - None - ); - assert_eq!(extract_tokens(Api::Anthropic, "not json"), None); - } - - /// One count present and the other missing still says what was - /// measured. - #[test] - fn a_half_count_is_reported_as_far_as_it_goes() { - assert_eq!( - extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#), - Some((31, 0)) - ); - } - #[test] fn extract_show_digest_finds_top_level() { let body = r#"{"digest":"sha256:abc"}"#; @@ -981,37 +841,6 @@ mod tests { assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); } - /// The counts of the answer reach the caller, not only the - /// extractor: the cost of a call belongs in the trail. - #[test] - fn chat_with_identity_carries_the_token_counts() { - let canned_chat = r#"{"message":{"content":"ja"}, - "prompt_eval_count":31,"eval_count":7}"# - .to_string(); - let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); - let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); - let p = ChatParams { - endpoint: "http://localhost:11434/api/chat", - ..params(Api::Ollama) - }; - let result = chat_with_identity(&client, &p).unwrap(); - assert_eq!(result.tokens, Some((31, 7))); - } - - /// A backend that counts nothing leaves the caller with nothing - /// to report, rather than with a zero. - #[test] - fn chat_with_identity_reports_no_counts_when_there_are_none() { - let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string(); - let client = MockClient::new(vec![Ok(canned_chat)]); - let p = ChatParams { - endpoint: "https://api.openai.com/v1/chat/completions", - ..params(Api::Openai) - }; - let result = chat_with_identity(&client, &p).unwrap(); - assert_eq!(result.tokens, None); - } - #[test] fn chat_with_identity_swallows_show_failure() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); @@ -1087,7 +916,6 @@ mod keep_alive_tests { system_prompt: "", prompt: "hello", keep_alive, - think: "", sampling: Sampling::default(), } } @@ -1171,7 +999,6 @@ mod sampling_tests { system_prompt: "", prompt: "hello", keep_alive: "", - think: "", sampling, } } @@ -1276,77 +1103,3 @@ mod sampling_tests { assert!(build_ollama_body(&p(Api::Ollama, s)).is_err()); } } - -#[cfg(test)] -mod think_tests { - #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] - use super::*; - - fn build_body(p: &ChatParams) -> Result { - match p.api { - Api::Ollama => build_ollama_body(p), - Api::Openai | Api::Vllm => build_openai_body(p), - Api::Anthropic => build_anthropic_body(p), - } - } - - fn p(api: Api, think: &str) -> ChatParams<'_> { - ChatParams { - api, - endpoint: "http://x", - model: "qwen3.8", - api_key: "", - system_prompt: "", - prompt: "hello", - keep_alive: "", - think, - sampling: Sampling::default(), - } - } - - /// Ollama lets qwen3-type models think by default; a call then - /// takes about 50 s instead of 10 s. - #[test] - fn ollama_gets_think_as_a_boolean() { - let body = build_ollama_body(&p(Api::Ollama, "false")).unwrap(); - assert!(body.contains(r#""think":false"#), "{body}"); - let body = build_ollama_body(&p(Api::Ollama, "True")).unwrap(); - assert!(body.contains(r#""think":true"#), "{body}"); - } - - #[test] - fn unset_leaves_every_body_unchanged() { - for api in [Api::Ollama, Api::Openai, Api::Vllm, Api::Anthropic] { - let body = build_body(&p(api, "")).unwrap(); - assert!(!body.contains("think"), "{body}"); - } - } - - #[test] - fn vllm_hands_the_switch_to_the_chat_template() { - let body = build_body(&p(Api::Vllm, "false")).unwrap(); - assert!( - body.contains(r#""chat_template_kwargs":{"enable_thinking":false}"#), - "{body}" - ); - } - - #[test] - fn a_switch_a_backend_cannot_honour_is_refused() { - assert!(matches!( - build_body(&p(Api::Openai, "false")), - Err(LlmError::UnsupportedApi(_)) - )); - assert!(matches!( - build_body(&p(Api::Anthropic, "true")), - Err(LlmError::UnsupportedApi(_)) - )); - assert!(matches!( - build_body(&p(Api::Ollama, "maybe")), - Err(LlmError::UnsupportedApi(_)) - )); - // Off is what Anthropic does anyway. - let body = build_body(&p(Api::Anthropic, "false")).unwrap(); - assert!(!body.contains("think"), "{body}"); - } -}