diff --git a/Cargo.lock b/Cargo.lock index 213beda..e2d8ab4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.3.0" +version = "0.3.1" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index a9ed91e..caceacd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.3.0" +version = "0.3.1" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.md b/MODULE.md index 0d100d9..f697dd1 100644 --- a/MODULE.md +++ b/MODULE.md @@ -21,6 +21,7 @@ Anthropic Messages API. | `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | | `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | | `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. | +| `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with ``. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | | `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | diff --git a/module.wasm b/module.wasm index d4243c2..c600c7f 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 0a471d7..62ea8af 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.3.0 +version: 0.3.1 # Capability provided by this module. provides: - capability: llm.chat - version: 0.3.0 + version: 0.3.1 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -129,6 +129,27 @@ inputs: großen lokalen Modell). Mit einer anderen api wird der Wert abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt, die die anderen Backends nicht erfüllen können. + think: + type: text + description: + en: | + Whether a reasoning model runs its thinking phase: true or + false; empty leaves the server default. Ollama lets + qwen3-type models think by default, which takes a call from + about 10 to about 50 seconds. Ollama receives think, vLLM + chat_template_kwargs.enable_thinking. The OpenAI wire has no + such switch, so a value is refused there; for Anthropic false + is accepted (extended thinking is off unless requested) and + true is refused. + de: | + Ob ein Reasoning-Modell seine Denkphase durchläuft: true oder + false; leer lässt die Voreinstellung des Servers. Ollama lässt + Modelle wie qwen3 standardmäßig denken, ein Aufruf dauert + dadurch rund 50 statt 10 Sekunden. Ollama bekommt think, vLLM + chat_template_kwargs.enable_thinking. Das OpenAI-Format kennt + keinen solchen Schalter, dort wird ein Wert abgelehnt; bei + Anthropic wird false angenommen (erweitertes Denken ist ohne + Anforderung aus) und true abgelehnt. system_prompt: type: text description: diff --git a/src/lib.rs b/src/lib.rs index 4c3a7aa..d4a432d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("keep_alive") .and_then(payload_text) .unwrap_or_default(); + // A flow may write `think: false` (JSON boolean) or "false". + let think = match inputs.get("think") { + Some(Payload::Text(s)) | Some(Payload::Json(s)) => s.clone(), + _ => String::new(), + }; let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default(); let temperature = sampling_input("temperature"); let top_p = sampling_input("top_p"); @@ -66,6 +71,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { system_prompt: &system_prompt, prompt: &prompt, keep_alive: &keep_alive, + think: &think, sampling: crate::llm::Sampling { temperature: &temperature, top_p: &top_p, diff --git a/src/llm.rs b/src/llm.rs index c435240..8fc1d6b 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -97,6 +97,11 @@ pub struct ChatParams<'a> { /// the server default, which unloads after five minutes and /// makes the next first request pay a 30 to 40 second reload. pub keep_alive: &'a str, + /// Whether a reasoning model runs its thinking phase: "true", + /// "false", or empty for the server default. Ollama lets + /// qwen3-type models think by default, which takes a call from + /// about 10 to about 50 seconds. + pub think: &'a str, /// Sampling controls. Empty means "leave the server default". /// /// `seed` and `temperature: 0` are what make a run repeatable, @@ -204,6 +209,8 @@ struct OllamaRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] keep_alive: Option>, #[serde(skip_serializing_if = "Option::is_none")] + think: Option, + #[serde(skip_serializing_if = "Option::is_none")] options: Option, } @@ -240,6 +247,27 @@ struct OpenAiRequest<'a> { top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] seed: Option, + /// vLLM only: passed to the chat template, where qwen3-type + /// templates read `enable_thinking`. + #[serde(skip_serializing_if = "Option::is_none")] + chat_template_kwargs: Option, +} + +#[derive(Serialize)] +struct ChatTemplateKwargs { + enable_thinking: bool, +} + +/// Parse the `think` input: empty is "server default". +pub fn parse_think(raw: &str) -> Result, LlmError> { + match raw.trim().to_ascii_lowercase().as_str() { + "" => Ok(None), + "true" => Ok(Some(true)), + "false" => Ok(Some(false)), + _ => Err(LlmError::UnsupportedApi(format!( + "think must be true or false, got {raw:?}" + ))), + } } #[derive(Serialize)] @@ -288,6 +316,7 @@ pub fn build_ollama_body(p: &ChatParams) -> Result { messages: build_messages(p), stream: false, keep_alive: parse_keep_alive(p.keep_alive), + think: parse_think(p.think)?, options: (!options.is_empty()).then_some(options), }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) @@ -306,6 +335,22 @@ pub fn build_openai_body(p: &ChatParams) -> Result { .to_string(), )); } + // vLLM hands the switch to the chat template. The OpenAI wire + // has no switch for it; refused rather than dropped, because a + // caller who sets it expects the call to take the time it asked + // for. + let think = parse_think(p.think)?; + let chat_template_kwargs = match (p.api, think) { + (_, None) => None, + (Api::Vllm, Some(enable_thinking)) => Some(ChatTemplateKwargs { enable_thinking }), + (_, Some(_)) => { + return Err(LlmError::UnsupportedApi( + "think has no equivalent on the OpenAI chat wire; use \ + api: ollama or vllm, or drop the input" + .to_string(), + )); + } + }; let req = OpenAiRequest { model: p.model, messages: build_messages(p), @@ -313,6 +358,7 @@ pub fn build_openai_body(p: &ChatParams) -> Result { temperature: parse_sampling("temperature", p.sampling.temperature, false)?, top_p: parse_sampling("top_p", p.sampling.top_p, false)?, seed: parse_sampling("seed", p.sampling.seed, true)?, + chat_template_kwargs, }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } @@ -331,6 +377,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result { .to_string(), )); } + // Extended thinking is off unless requested, so "false" is what + // the server does anyway. Turning it on needs a token budget this + // module does not take yet. + if parse_think(p.think)? == Some(true) { + return Err(LlmError::UnsupportedApi( + "think: true is not supported for the Anthropic Messages API \ + here (extended thinking needs a token budget); use false or \ + drop the input" + .to_string(), + )); + } let req = AnthropicRequest { model: p.model, max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -564,6 +621,7 @@ mod tests { system_prompt: "", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), } } @@ -654,6 +712,7 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), }; let body = build_openai_body(&p).expect("body builds"); @@ -725,6 +784,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -747,6 +807,7 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + think: "", sampling: Sampling::default(), }; let body = build_anthropic_body(&p).expect("body builds"); @@ -829,6 +890,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + think: "", sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); @@ -1025,6 +1087,7 @@ mod keep_alive_tests { system_prompt: "", prompt: "hello", keep_alive, + think: "", sampling: Sampling::default(), } } @@ -1108,6 +1171,7 @@ mod sampling_tests { system_prompt: "", prompt: "hello", keep_alive: "", + think: "", sampling, } } @@ -1212,3 +1276,77 @@ mod sampling_tests { assert!(build_ollama_body(&p(Api::Ollama, s)).is_err()); } } + +#[cfg(test)] +mod think_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + + fn build_body(p: &ChatParams) -> Result { + match p.api { + Api::Ollama => build_ollama_body(p), + Api::Openai | Api::Vllm => build_openai_body(p), + Api::Anthropic => build_anthropic_body(p), + } + } + + fn p(api: Api, think: &str) -> ChatParams<'_> { + ChatParams { + api, + endpoint: "http://x", + model: "qwen3.8", + api_key: "", + system_prompt: "", + prompt: "hello", + keep_alive: "", + think, + sampling: Sampling::default(), + } + } + + /// Ollama lets qwen3-type models think by default; a call then + /// takes about 50 s instead of 10 s. + #[test] + fn ollama_gets_think_as_a_boolean() { + let body = build_ollama_body(&p(Api::Ollama, "false")).unwrap(); + assert!(body.contains(r#""think":false"#), "{body}"); + let body = build_ollama_body(&p(Api::Ollama, "True")).unwrap(); + assert!(body.contains(r#""think":true"#), "{body}"); + } + + #[test] + fn unset_leaves_every_body_unchanged() { + for api in [Api::Ollama, Api::Openai, Api::Vllm, Api::Anthropic] { + let body = build_body(&p(api, "")).unwrap(); + assert!(!body.contains("think"), "{body}"); + } + } + + #[test] + fn vllm_hands_the_switch_to_the_chat_template() { + let body = build_body(&p(Api::Vllm, "false")).unwrap(); + assert!( + body.contains(r#""chat_template_kwargs":{"enable_thinking":false}"#), + "{body}" + ); + } + + #[test] + fn a_switch_a_backend_cannot_honour_is_refused() { + assert!(matches!( + build_body(&p(Api::Openai, "false")), + Err(LlmError::UnsupportedApi(_)) + )); + assert!(matches!( + build_body(&p(Api::Anthropic, "true")), + Err(LlmError::UnsupportedApi(_)) + )); + assert!(matches!( + build_body(&p(Api::Ollama, "maybe")), + Err(LlmError::UnsupportedApi(_)) + )); + // Off is what Anthropic does anyway. + let body = build_body(&p(Api::Anthropic, "false")).unwrap(); + assert!(!body.contains("think"), "{body}"); + } +}