diff --git a/Cargo.lock b/Cargo.lock index c510695..e2d8ab4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.2.1" +version = "0.3.1" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6391d14..caceacd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.2.1" +version = "0.3.1" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.de.md b/MODULE.de.md index 21c2d00..8e1bb5b 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -10,7 +10,7 @@ die Anthropic Messages API. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.3.0` ## Eingaben @@ -40,6 +40,8 @@ die Anthropic Messages API. | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | | `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | +| `tokens_prompt` | text | Was die Anfrage gekostet hat, nach der Zählung des Backends. Leer, wenn die Antwort keine Zählung enthielt. | +| `tokens_completion` | text | Was die Antwort gekostet hat, gleiche Regel. Ollama zählt in `eval_count`, das OpenAI-Format in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Zusammen beantworten die drei `model_*`-Ausgaben die Audit- Frage: „Welches genaue Modell hat diese Antwort erzeugt?" — diff --git a/MODULE.md b/MODULE.md index 29a511e..f697dd1 100644 --- a/MODULE.md +++ b/MODULE.md @@ -10,7 +10,7 @@ Anthropic Messages API. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.3.0` ## Inputs @@ -21,6 +21,7 @@ Anthropic Messages API. | `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | | `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | | `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. | +| `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with ``. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | | `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | @@ -40,6 +41,8 @@ Anthropic Messages API. | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | | `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | +| `tokens_prompt` | text | What the prompt cost, as the backend counted it. Empty when the answer carried no count. | +| `tokens_completion` | text | What the answer cost, same rule. Ollama counts in `eval_count`, the OpenAI shape in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Together the three `model_*` outputs answer the audit question: "which exact model produced this response?" — that diff --git a/module.wasm b/module.wasm index b7e865b..c600c7f 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 43727f8..62ea8af 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.2.2 +version: 0.3.1 # Capability provided by this module. provides: - capability: llm.chat - version: 0.2.2 + version: 0.3.1 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -129,6 +129,27 @@ inputs: großen lokalen Modell). Mit einer anderen api wird der Wert abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt, die die anderen Backends nicht erfüllen können. + think: + type: text + description: + en: | + Whether a reasoning model runs its thinking phase: true or + false; empty leaves the server default. Ollama lets + qwen3-type models think by default, which takes a call from + about 10 to about 50 seconds. Ollama receives think, vLLM + chat_template_kwargs.enable_thinking. The OpenAI wire has no + such switch, so a value is refused there; for Anthropic false + is accepted (extended thinking is off unless requested) and + true is refused. + de: | + Ob ein Reasoning-Modell seine Denkphase durchläuft: true oder + false; leer lässt die Voreinstellung des Servers. Ollama lässt + Modelle wie qwen3 standardmäßig denken, ein Aufruf dauert + dadurch rund 50 statt 10 Sekunden. Ollama bekommt think, vLLM + chat_template_kwargs.enable_thinking. Das OpenAI-Format kennt + keinen solchen Schalter, dort wird ein Wert abgelehnt; bei + Anthropic wird false angenommen (erweitertes Denken ist ohne + Anforderung aus) und true abgelehnt. system_prompt: type: text description: @@ -171,6 +192,32 @@ outputs: keine Digest-API bereit). Zusammen mit model_endpoint und model_name beantwortet das die Audit-Frage "welches genaue Modell hat diese Antwort erzeugt?" + tokens_prompt: + type: text + description: + en: | + How many tokens the prompt cost, as the backend counted it. + Empty when the answer carried no count: a zero would read + like a measurement nobody took. The hub lifts this field + into the audit log automatically. + de: | + Wie viele Token die Anfrage gekostet hat, nach der Zählung + des Backends. Leer, wenn die Antwort keine Zählung enthielt: + eine Null würde wie eine Messung aussehen, die niemand + gemacht hat. Der Hub hebt dieses Feld selbst in den Prüfpfad. + tokens_completion: + type: text + description: + en: | + How many tokens the answer cost, same rules as + tokens_prompt. Ollama counts in `eval_count`, the OpenAI + shape in `usage.completion_tokens`, Anthropic in + `usage.output_tokens`. + de: | + Wie viele Token die Antwort gekostet hat, gleiche Regeln wie + tokens_prompt. Ollama zählt in `eval_count`, das + OpenAI-Format in `usage.completion_tokens`, Anthropic in + `usage.output_tokens`. # Permissions required. # diff --git a/src/lib.rs b/src/lib.rs index 4b31f50..d4a432d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("keep_alive") .and_then(payload_text) .unwrap_or_default(); + // A flow may write `think: false` (JSON boolean) or "false". + let think = match inputs.get("think") { + Some(Payload::Text(s)) | Some(Payload::Json(s)) => s.clone(), + _ => String::new(), + }; let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default(); let temperature = sampling_input("temperature"); let top_p = sampling_input("top_p"); @@ -66,6 +71,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { system_prompt: &system_prompt, prompt: &prompt, keep_alive: &keep_alive, + think: &think, sampling: crate::llm::Sampling { temperature: &temperature, top_p: &top_p, @@ -76,11 +82,20 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let result = crate::llm::chat_with_identity(&client, ¶ms) .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; + // The two token counts are the cost of this call, as the backend + // counted it. They stay empty where the answer carried no count, + // because an invented zero would look like a measurement. + let (tokens_prompt, tokens_completion) = match result.tokens { + Some((prompt, completion)) => (prompt.to_string(), completion.to_string()), + None => (String::new(), String::new()), + }; Ok(Outputs::new() .with_text("response", result.response) .with_text("model_endpoint", endpoint) .with_text("model_name", model) - .with_text("model_digest", result.model_digest.unwrap_or_default())) + .with_text("model_digest", result.model_digest.unwrap_or_default()) + .with_text("tokens_prompt", tokens_prompt) + .with_text("tokens_completion", tokens_completion)) } fn payload_text(p: &Payload) -> Option { diff --git a/src/llm.rs b/src/llm.rs index bb569e4..8fc1d6b 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -97,6 +97,11 @@ pub struct ChatParams<'a> { /// the server default, which unloads after five minutes and /// makes the next first request pay a 30 to 40 second reload. pub keep_alive: &'a str, + /// Whether a reasoning model runs its thinking phase: "true", + /// "false", or empty for the server default. Ollama lets + /// qwen3-type models think by default, which takes a call from + /// about 10 to about 50 seconds. + pub think: &'a str, /// Sampling controls. Empty means "leave the server default". /// /// `seed` and `temperature: 0` are what make a run repeatable, @@ -204,6 +209,8 @@ struct OllamaRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] keep_alive: Option>, #[serde(skip_serializing_if = "Option::is_none")] + think: Option, + #[serde(skip_serializing_if = "Option::is_none")] options: Option, } @@ -240,6 +247,27 @@ struct OpenAiRequest<'a> { top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] seed: Option, + /// vLLM only: passed to the chat template, where qwen3-type + /// templates read `enable_thinking`. + #[serde(skip_serializing_if = "Option::is_none")] + chat_template_kwargs: Option, +} + +#[derive(Serialize)] +struct ChatTemplateKwargs { + enable_thinking: bool, +} + +/// Parse the `think` input: empty is "server default". +pub fn parse_think(raw: &str) -> Result, LlmError> { + match raw.trim().to_ascii_lowercase().as_str() { + "" => Ok(None), + "true" => Ok(Some(true)), + "false" => Ok(Some(false)), + _ => Err(LlmError::UnsupportedApi(format!( + "think must be true or false, got {raw:?}" + ))), + } } #[derive(Serialize)] @@ -288,6 +316,7 @@ pub fn build_ollama_body(p: &ChatParams) -> Result { messages: build_messages(p), stream: false, keep_alive: parse_keep_alive(p.keep_alive), + think: parse_think(p.think)?, options: (!options.is_empty()).then_some(options), }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) @@ -306,6 +335,22 @@ pub fn build_openai_body(p: &ChatParams) -> Result { .to_string(), )); } + // vLLM hands the switch to the chat template. The OpenAI wire + // has no switch for it; refused rather than dropped, because a + // caller who sets it expects the call to take the time it asked + // for. + let think = parse_think(p.think)?; + let chat_template_kwargs = match (p.api, think) { + (_, None) => None, + (Api::Vllm, Some(enable_thinking)) => Some(ChatTemplateKwargs { enable_thinking }), + (_, Some(_)) => { + return Err(LlmError::UnsupportedApi( + "think has no equivalent on the OpenAI chat wire; use \ + api: ollama or vllm, or drop the input" + .to_string(), + )); + } + }; let req = OpenAiRequest { model: p.model, messages: build_messages(p), @@ -313,6 +358,7 @@ pub fn build_openai_body(p: &ChatParams) -> Result { temperature: parse_sampling("temperature", p.sampling.temperature, false)?, top_p: parse_sampling("top_p", p.sampling.top_p, false)?, seed: parse_sampling("seed", p.sampling.seed, true)?, + chat_template_kwargs, }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } @@ -331,6 +377,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result { .to_string(), )); } + // Extended thinking is off unless requested, so "false" is what + // the server does anyway. Turning it on needs a token budget this + // module does not take yet. + if parse_think(p.think)? == Some(true) { + return Err(LlmError::UnsupportedApi( + "think: true is not supported for the Anthropic Messages API \ + here (extended thinking needs a token budget); use false or \ + drop the input" + .to_string(), + )); + } let req = AnthropicRequest { model: p.model, max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -393,6 +450,41 @@ pub fn extract_anthropic_content(body: &str) -> Result { pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, + /// How many tokens the call cost, as the backend counted them: + /// (prompt, completion). `None` where the answer carries no + /// count, which is honest about a number nobody measured. + pub tokens: Option<(u64, u64)>, +} + +/// The token counts of one answer, per wire format. +/// +/// Three dialects, three spellings of the same two numbers: Ollama +/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape +/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic +/// in `usage.input_tokens` and `usage.output_tokens`. A missing +/// count is `None` rather than a zero, because zero is a +/// measurement and this is the absence of one. +pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> { + let v: serde_json::Value = serde_json::from_str(body).ok()?; + let pair = match api { + Api::Ollama => ( + v.get("prompt_eval_count").and_then(|n| n.as_u64()), + v.get("eval_count").and_then(|n| n.as_u64()), + ), + Api::Openai | Api::Vllm => ( + v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()), + v.pointer("/usage/completion_tokens") + .and_then(|n| n.as_u64()), + ), + Api::Anthropic => ( + v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()), + v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()), + ), + }; + match pair { + (None, None) => None, + (prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))), + } } /// Run one chat call in the selected wire format AND, for Ollama, @@ -429,9 +521,11 @@ pub fn chat_with_identity( Api::Ollama => probe_model_digest(client, p), Api::Openai | Api::Vllm | Api::Anthropic => None, }; + let tokens = extract_tokens(p.api, &response_body); Ok(ChatWithIdentity { response, model_digest, + tokens, }) } @@ -527,6 +621,7 @@ mod tests { system_prompt: "", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), } } @@ -617,6 +712,7 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), }; let body = build_openai_body(&p).expect("body builds"); @@ -688,6 +784,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -710,6 +807,7 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), }; let body = build_anthropic_body(&p).expect("body builds"); @@ -792,6 +890,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -815,6 +914,47 @@ mod tests { assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); } + /// Three dialects, three spellings of the same two numbers. + #[test] + fn every_dialect_reports_its_token_counts() { + let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#; + assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7))); + + let openai = r#"{"choices":[{"message":{"content":"ja"}}], + "usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#; + assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7))); + assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7))); + + let anthropic = r#"{"content":[{"text":"ja"}], + "usage":{"input_tokens":31,"output_tokens":7}}"#; + assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7))); + } + + /// A backend that counts nothing leaves the fields empty, rather + /// than reporting a zero nobody measured. + #[test] + fn an_answer_without_counts_reports_none() { + assert_eq!( + extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#), + None + ); + assert_eq!( + extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#), + None + ); + assert_eq!(extract_tokens(Api::Anthropic, "not json"), None); + } + + /// One count present and the other missing still says what was + /// measured. + #[test] + fn a_half_count_is_reported_as_far_as_it_goes() { + assert_eq!( + extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#), + Some((31, 0)) + ); + } + #[test] fn extract_show_digest_finds_top_level() { let body = r#"{"digest":"sha256:abc"}"#; @@ -841,6 +981,37 @@ mod tests { assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); } + /// The counts of the answer reach the caller, not only the + /// extractor: the cost of a call belongs in the trail. + #[test] + fn chat_with_identity_carries_the_token_counts() { + let canned_chat = r#"{"message":{"content":"ja"}, + "prompt_eval_count":31,"eval_count":7}"# + .to_string(); + let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); + let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); + let p = ChatParams { + endpoint: "http://localhost:11434/api/chat", + ..params(Api::Ollama) + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.tokens, Some((31, 7))); + } + + /// A backend that counts nothing leaves the caller with nothing + /// to report, rather than with a zero. + #[test] + fn chat_with_identity_reports_no_counts_when_there_are_none() { + let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string(); + let client = MockClient::new(vec![Ok(canned_chat)]); + let p = ChatParams { + endpoint: "https://api.openai.com/v1/chat/completions", + ..params(Api::Openai) + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.tokens, None); + } + #[test] fn chat_with_identity_swallows_show_failure() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); @@ -916,6 +1087,7 @@ mod keep_alive_tests { system_prompt: "", prompt: "hello", keep_alive, + think: "", sampling: Sampling::default(), } } @@ -999,6 +1171,7 @@ mod sampling_tests { system_prompt: "", prompt: "hello", keep_alive: "", + think: "", sampling, } } @@ -1103,3 +1276,77 @@ mod sampling_tests { assert!(build_ollama_body(&p(Api::Ollama, s)).is_err()); } } + +#[cfg(test)] +mod think_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + + fn build_body(p: &ChatParams) -> Result { + match p.api { + Api::Ollama => build_ollama_body(p), + Api::Openai | Api::Vllm => build_openai_body(p), + Api::Anthropic => build_anthropic_body(p), + } + } + + fn p(api: Api, think: &str) -> ChatParams<'_> { + ChatParams { + api, + endpoint: "http://x", + model: "qwen3.8", + api_key: "", + system_prompt: "", + prompt: "hello", + keep_alive: "", + think, + sampling: Sampling::default(), + } + } + + /// Ollama lets qwen3-type models think by default; a call then + /// takes about 50 s instead of 10 s. + #[test] + fn ollama_gets_think_as_a_boolean() { + let body = build_ollama_body(&p(Api::Ollama, "false")).unwrap(); + assert!(body.contains(r#""think":false"#), "{body}"); + let body = build_ollama_body(&p(Api::Ollama, "True")).unwrap(); + assert!(body.contains(r#""think":true"#), "{body}"); + } + + #[test] + fn unset_leaves_every_body_unchanged() { + for api in [Api::Ollama, Api::Openai, Api::Vllm, Api::Anthropic] { + let body = build_body(&p(api, "")).unwrap(); + assert!(!body.contains("think"), "{body}"); + } + } + + #[test] + fn vllm_hands_the_switch_to_the_chat_template() { + let body = build_body(&p(Api::Vllm, "false")).unwrap(); + assert!( + body.contains(r#""chat_template_kwargs":{"enable_thinking":false}"#), + "{body}" + ); + } + + #[test] + fn a_switch_a_backend_cannot_honour_is_refused() { + assert!(matches!( + build_body(&p(Api::Openai, "false")), + Err(LlmError::UnsupportedApi(_)) + )); + assert!(matches!( + build_body(&p(Api::Anthropic, "true")), + Err(LlmError::UnsupportedApi(_)) + )); + assert!(matches!( + build_body(&p(Api::Ollama, "maybe")), + Err(LlmError::UnsupportedApi(_)) + )); + // Off is what Anthropic does anyway. + let body = build_body(&p(Api::Anthropic, "false")).unwrap(); + assert!(!body.contains("think"), "{body}"); + } +}