feat: think input switches the thinking phase of reasoning models, 0.3.1
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
Ollama gets think, vLLM chat_template_kwargs.enable_thinking. Refused where the backend has no such switch. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
d5b000e18e
commit
6455a7457e
7 changed files with 170 additions and 4 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "llm_chat"
|
name = "llm_chat"
|
||||||
version = "0.3.0"
|
version = "0.3.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chain-module-sdk",
|
"chain-module-sdk",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "llm_chat"
|
name = "llm_chat"
|
||||||
version = "0.3.0"
|
version = "0.3.1"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
publish = false
|
publish = false
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ Anthropic Messages API.
|
||||||
| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). |
|
| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). |
|
||||||
| `api_key` | text | Optional API key. Never appears in outputs, logs, or events. |
|
| `api_key` | text | Optional API key. Never appears in outputs, logs, or events. |
|
||||||
| `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. |
|
| `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. |
|
||||||
|
| `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with `</think>`. |
|
||||||
| `system_prompt` | text | Optional system message. Empty = use the model's default. |
|
| `system_prompt` | text | Optional system message. Empty = use the model's default. |
|
||||||
| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. |
|
| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. |
|
||||||
|
|
||||||
|
|
|
||||||
BIN
module.wasm
BIN
module.wasm
Binary file not shown.
25
module.yaml
25
module.yaml
|
|
@ -1,12 +1,12 @@
|
||||||
schema_version: 3
|
schema_version: 3
|
||||||
provider: chain
|
provider: chain
|
||||||
name: llm-chat
|
name: llm-chat
|
||||||
version: 0.3.0
|
version: 0.3.1
|
||||||
|
|
||||||
# Capability provided by this module.
|
# Capability provided by this module.
|
||||||
provides:
|
provides:
|
||||||
- capability: llm.chat
|
- capability: llm.chat
|
||||||
version: 0.3.0
|
version: 0.3.1
|
||||||
|
|
||||||
# Declared inputs in verbose v3 form. Studio renders the
|
# Declared inputs in verbose v3 form. Studio renders the
|
||||||
# description in the active locale as a tooltip on each
|
# description in the active locale as a tooltip on each
|
||||||
|
|
@ -129,6 +129,27 @@ inputs:
|
||||||
großen lokalen Modell). Mit einer anderen api wird der Wert
|
großen lokalen Modell). Mit einer anderen api wird der Wert
|
||||||
abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt,
|
abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt,
|
||||||
die die anderen Backends nicht erfüllen können.
|
die die anderen Backends nicht erfüllen können.
|
||||||
|
think:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
Whether a reasoning model runs its thinking phase: true or
|
||||||
|
false; empty leaves the server default. Ollama lets
|
||||||
|
qwen3-type models think by default, which takes a call from
|
||||||
|
about 10 to about 50 seconds. Ollama receives think, vLLM
|
||||||
|
chat_template_kwargs.enable_thinking. The OpenAI wire has no
|
||||||
|
such switch, so a value is refused there; for Anthropic false
|
||||||
|
is accepted (extended thinking is off unless requested) and
|
||||||
|
true is refused.
|
||||||
|
de: |
|
||||||
|
Ob ein Reasoning-Modell seine Denkphase durchläuft: true oder
|
||||||
|
false; leer lässt die Voreinstellung des Servers. Ollama lässt
|
||||||
|
Modelle wie qwen3 standardmäßig denken, ein Aufruf dauert
|
||||||
|
dadurch rund 50 statt 10 Sekunden. Ollama bekommt think, vLLM
|
||||||
|
chat_template_kwargs.enable_thinking. Das OpenAI-Format kennt
|
||||||
|
keinen solchen Schalter, dort wird ein Wert abgelehnt; bei
|
||||||
|
Anthropic wird false angenommen (erweitertes Denken ist ohne
|
||||||
|
Anforderung aus) und true abgelehnt.
|
||||||
system_prompt:
|
system_prompt:
|
||||||
type: text
|
type: text
|
||||||
description:
|
description:
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
.get("keep_alive")
|
.get("keep_alive")
|
||||||
.and_then(payload_text)
|
.and_then(payload_text)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
// A flow may write `think: false` (JSON boolean) or "false".
|
||||||
|
let think = match inputs.get("think") {
|
||||||
|
Some(Payload::Text(s)) | Some(Payload::Json(s)) => s.clone(),
|
||||||
|
_ => String::new(),
|
||||||
|
};
|
||||||
let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default();
|
let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default();
|
||||||
let temperature = sampling_input("temperature");
|
let temperature = sampling_input("temperature");
|
||||||
let top_p = sampling_input("top_p");
|
let top_p = sampling_input("top_p");
|
||||||
|
|
@ -66,6 +71,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
system_prompt: &system_prompt,
|
system_prompt: &system_prompt,
|
||||||
prompt: &prompt,
|
prompt: &prompt,
|
||||||
keep_alive: &keep_alive,
|
keep_alive: &keep_alive,
|
||||||
|
think: &think,
|
||||||
sampling: crate::llm::Sampling {
|
sampling: crate::llm::Sampling {
|
||||||
temperature: &temperature,
|
temperature: &temperature,
|
||||||
top_p: &top_p,
|
top_p: &top_p,
|
||||||
|
|
|
||||||
138
src/llm.rs
138
src/llm.rs
|
|
@ -97,6 +97,11 @@ pub struct ChatParams<'a> {
|
||||||
/// the server default, which unloads after five minutes and
|
/// the server default, which unloads after five minutes and
|
||||||
/// makes the next first request pay a 30 to 40 second reload.
|
/// makes the next first request pay a 30 to 40 second reload.
|
||||||
pub keep_alive: &'a str,
|
pub keep_alive: &'a str,
|
||||||
|
/// Whether a reasoning model runs its thinking phase: "true",
|
||||||
|
/// "false", or empty for the server default. Ollama lets
|
||||||
|
/// qwen3-type models think by default, which takes a call from
|
||||||
|
/// about 10 to about 50 seconds.
|
||||||
|
pub think: &'a str,
|
||||||
/// Sampling controls. Empty means "leave the server default".
|
/// Sampling controls. Empty means "leave the server default".
|
||||||
///
|
///
|
||||||
/// `seed` and `temperature: 0` are what make a run repeatable,
|
/// `seed` and `temperature: 0` are what make a run repeatable,
|
||||||
|
|
@ -204,6 +209,8 @@ struct OllamaRequest<'a> {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
keep_alive: Option<KeepAlive<'a>>,
|
keep_alive: Option<KeepAlive<'a>>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
think: Option<bool>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
options: Option<OllamaOptions>,
|
options: Option<OllamaOptions>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -240,6 +247,27 @@ struct OpenAiRequest<'a> {
|
||||||
top_p: Option<SamplingValue>,
|
top_p: Option<SamplingValue>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
seed: Option<SamplingValue>,
|
seed: Option<SamplingValue>,
|
||||||
|
/// vLLM only: passed to the chat template, where qwen3-type
|
||||||
|
/// templates read `enable_thinking`.
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
chat_template_kwargs: Option<ChatTemplateKwargs>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct ChatTemplateKwargs {
|
||||||
|
enable_thinking: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the `think` input: empty is "server default".
|
||||||
|
pub fn parse_think(raw: &str) -> Result<Option<bool>, LlmError> {
|
||||||
|
match raw.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"" => Ok(None),
|
||||||
|
"true" => Ok(Some(true)),
|
||||||
|
"false" => Ok(Some(false)),
|
||||||
|
_ => Err(LlmError::UnsupportedApi(format!(
|
||||||
|
"think must be true or false, got {raw:?}"
|
||||||
|
))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -288,6 +316,7 @@ pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
messages: build_messages(p),
|
messages: build_messages(p),
|
||||||
stream: false,
|
stream: false,
|
||||||
keep_alive: parse_keep_alive(p.keep_alive),
|
keep_alive: parse_keep_alive(p.keep_alive),
|
||||||
|
think: parse_think(p.think)?,
|
||||||
options: (!options.is_empty()).then_some(options),
|
options: (!options.is_empty()).then_some(options),
|
||||||
};
|
};
|
||||||
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||||
|
|
@ -306,6 +335,22 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
.to_string(),
|
.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// vLLM hands the switch to the chat template. The OpenAI wire
|
||||||
|
// has no switch for it; refused rather than dropped, because a
|
||||||
|
// caller who sets it expects the call to take the time it asked
|
||||||
|
// for.
|
||||||
|
let think = parse_think(p.think)?;
|
||||||
|
let chat_template_kwargs = match (p.api, think) {
|
||||||
|
(_, None) => None,
|
||||||
|
(Api::Vllm, Some(enable_thinking)) => Some(ChatTemplateKwargs { enable_thinking }),
|
||||||
|
(_, Some(_)) => {
|
||||||
|
return Err(LlmError::UnsupportedApi(
|
||||||
|
"think has no equivalent on the OpenAI chat wire; use \
|
||||||
|
api: ollama or vllm, or drop the input"
|
||||||
|
.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
let req = OpenAiRequest {
|
let req = OpenAiRequest {
|
||||||
model: p.model,
|
model: p.model,
|
||||||
messages: build_messages(p),
|
messages: build_messages(p),
|
||||||
|
|
@ -313,6 +358,7 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||||
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||||
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||||
|
chat_template_kwargs,
|
||||||
};
|
};
|
||||||
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||||
}
|
}
|
||||||
|
|
@ -331,6 +377,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
.to_string(),
|
.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// Extended thinking is off unless requested, so "false" is what
|
||||||
|
// the server does anyway. Turning it on needs a token budget this
|
||||||
|
// module does not take yet.
|
||||||
|
if parse_think(p.think)? == Some(true) {
|
||||||
|
return Err(LlmError::UnsupportedApi(
|
||||||
|
"think: true is not supported for the Anthropic Messages API \
|
||||||
|
here (extended thinking needs a token budget); use false or \
|
||||||
|
drop the input"
|
||||||
|
.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let req = AnthropicRequest {
|
let req = AnthropicRequest {
|
||||||
model: p.model,
|
model: p.model,
|
||||||
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||||
|
|
@ -564,6 +621,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -654,6 +712,7 @@ mod tests {
|
||||||
system_prompt: "be terse",
|
system_prompt: "be terse",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let body = build_openai_body(&p).expect("body builds");
|
let body = build_openai_body(&p).expect("body builds");
|
||||||
|
|
@ -725,6 +784,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hi",
|
prompt: "hi",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let result = chat_with_identity(&client, &p).unwrap();
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
|
|
@ -747,6 +807,7 @@ mod tests {
|
||||||
system_prompt: "be terse",
|
system_prompt: "be terse",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let body = build_anthropic_body(&p).expect("body builds");
|
let body = build_anthropic_body(&p).expect("body builds");
|
||||||
|
|
@ -829,6 +890,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hi",
|
prompt: "hi",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let result = chat_with_identity(&client, &p).unwrap();
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
|
|
@ -1025,6 +1087,7 @@ mod keep_alive_tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive,
|
keep_alive,
|
||||||
|
think: "",
|
||||||
sampling: Sampling::default(),
|
sampling: Sampling::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1108,6 +1171,7 @@ mod sampling_tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
think: "",
|
||||||
sampling,
|
sampling,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -1212,3 +1276,77 @@ mod sampling_tests {
|
||||||
assert!(build_ollama_body(&p(Api::Ollama, s)).is_err());
|
assert!(build_ollama_body(&p(Api::Ollama, s)).is_err());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod think_tests {
|
||||||
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn build_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
|
match p.api {
|
||||||
|
Api::Ollama => build_ollama_body(p),
|
||||||
|
Api::Openai | Api::Vllm => build_openai_body(p),
|
||||||
|
Api::Anthropic => build_anthropic_body(p),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn p(api: Api, think: &str) -> ChatParams<'_> {
|
||||||
|
ChatParams {
|
||||||
|
api,
|
||||||
|
endpoint: "http://x",
|
||||||
|
model: "qwen3.8",
|
||||||
|
api_key: "",
|
||||||
|
system_prompt: "",
|
||||||
|
prompt: "hello",
|
||||||
|
keep_alive: "",
|
||||||
|
think,
|
||||||
|
sampling: Sampling::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ollama lets qwen3-type models think by default; a call then
|
||||||
|
/// takes about 50 s instead of 10 s.
|
||||||
|
#[test]
|
||||||
|
fn ollama_gets_think_as_a_boolean() {
|
||||||
|
let body = build_ollama_body(&p(Api::Ollama, "false")).unwrap();
|
||||||
|
assert!(body.contains(r#""think":false"#), "{body}");
|
||||||
|
let body = build_ollama_body(&p(Api::Ollama, "True")).unwrap();
|
||||||
|
assert!(body.contains(r#""think":true"#), "{body}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unset_leaves_every_body_unchanged() {
|
||||||
|
for api in [Api::Ollama, Api::Openai, Api::Vllm, Api::Anthropic] {
|
||||||
|
let body = build_body(&p(api, "")).unwrap();
|
||||||
|
assert!(!body.contains("think"), "{body}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn vllm_hands_the_switch_to_the_chat_template() {
|
||||||
|
let body = build_body(&p(Api::Vllm, "false")).unwrap();
|
||||||
|
assert!(
|
||||||
|
body.contains(r#""chat_template_kwargs":{"enable_thinking":false}"#),
|
||||||
|
"{body}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_switch_a_backend_cannot_honour_is_refused() {
|
||||||
|
assert!(matches!(
|
||||||
|
build_body(&p(Api::Openai, "false")),
|
||||||
|
Err(LlmError::UnsupportedApi(_))
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
build_body(&p(Api::Anthropic, "true")),
|
||||||
|
Err(LlmError::UnsupportedApi(_))
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
build_body(&p(Api::Ollama, "maybe")),
|
||||||
|
Err(LlmError::UnsupportedApi(_))
|
||||||
|
));
|
||||||
|
// Off is what Anthropic does anyway.
|
||||||
|
let body = build_body(&p(Api::Anthropic, "false")).unwrap();
|
||||||
|
assert!(!body.contains("think"), "{body}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue