Compare commits
2 commits
a6fe962916
...
6455a7457e
| Author | SHA1 | Date | |
|---|---|---|---|
| 6455a7457e | |||
| d5b000e18e |
8 changed files with 321 additions and 7 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545"
|
|||
|
||||
[[package]]
|
||||
name = "llm_chat"
|
||||
version = "0.2.1"
|
||||
version = "0.3.1"
|
||||
dependencies = [
|
||||
"chain-module-sdk",
|
||||
"serde",
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
|
||||
[package]
|
||||
name = "llm_chat"
|
||||
version = "0.2.1"
|
||||
version = "0.3.1"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
publish = false
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ die Anthropic Messages API.
|
|||
|
||||
## Capability
|
||||
|
||||
- `llm.chat@0.2.0`
|
||||
- `llm.chat@0.3.0`
|
||||
|
||||
## Eingaben
|
||||
|
||||
|
|
@ -40,6 +40,8 @@ die Anthropic Messages API.
|
|||
| `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. |
|
||||
| `model_name` | text | Modell-ID wie an die LLM-API gesendet. |
|
||||
| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. |
|
||||
| `tokens_prompt` | text | Was die Anfrage gekostet hat, nach der Zählung des Backends. Leer, wenn die Antwort keine Zählung enthielt. |
|
||||
| `tokens_completion` | text | Was die Antwort gekostet hat, gleiche Regel. Ollama zählt in `eval_count`, das OpenAI-Format in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. |
|
||||
|
||||
Zusammen beantworten die drei `model_*`-Ausgaben die Audit-
|
||||
Frage: „Welches genaue Modell hat diese Antwort erzeugt?" —
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ Anthropic Messages API.
|
|||
|
||||
## Capability
|
||||
|
||||
- `llm.chat@0.2.0`
|
||||
- `llm.chat@0.3.0`
|
||||
|
||||
## Inputs
|
||||
|
||||
|
|
@ -21,6 +21,7 @@ Anthropic Messages API.
|
|||
| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). |
|
||||
| `api_key` | text | Optional API key. Never appears in outputs, logs, or events. |
|
||||
| `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. |
|
||||
| `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with `</think>`. |
|
||||
| `system_prompt` | text | Optional system message. Empty = use the model's default. |
|
||||
| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. |
|
||||
|
||||
|
|
@ -40,6 +41,8 @@ Anthropic Messages API.
|
|||
| `model_endpoint` | text | The endpoint the response was generated against. |
|
||||
| `model_name` | text | The model identifier as supplied to the LLM API. |
|
||||
| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. |
|
||||
| `tokens_prompt` | text | What the prompt cost, as the backend counted it. Empty when the answer carried no count. |
|
||||
| `tokens_completion` | text | What the answer cost, same rule. Ollama counts in `eval_count`, the OpenAI shape in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. |
|
||||
|
||||
Together the three `model_*` outputs answer the audit
|
||||
question: "which exact model produced this response?" — that
|
||||
|
|
|
|||
BIN
module.wasm
BIN
module.wasm
Binary file not shown.
51
module.yaml
51
module.yaml
|
|
@ -1,12 +1,12 @@
|
|||
schema_version: 3
|
||||
provider: chain
|
||||
name: llm-chat
|
||||
version: 0.2.2
|
||||
version: 0.3.1
|
||||
|
||||
# Capability provided by this module.
|
||||
provides:
|
||||
- capability: llm.chat
|
||||
version: 0.2.2
|
||||
version: 0.3.1
|
||||
|
||||
# Declared inputs in verbose v3 form. Studio renders the
|
||||
# description in the active locale as a tooltip on each
|
||||
|
|
@ -129,6 +129,27 @@ inputs:
|
|||
großen lokalen Modell). Mit einer anderen api wird der Wert
|
||||
abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt,
|
||||
die die anderen Backends nicht erfüllen können.
|
||||
think:
|
||||
type: text
|
||||
description:
|
||||
en: |
|
||||
Whether a reasoning model runs its thinking phase: true or
|
||||
false; empty leaves the server default. Ollama lets
|
||||
qwen3-type models think by default, which takes a call from
|
||||
about 10 to about 50 seconds. Ollama receives think, vLLM
|
||||
chat_template_kwargs.enable_thinking. The OpenAI wire has no
|
||||
such switch, so a value is refused there; for Anthropic false
|
||||
is accepted (extended thinking is off unless requested) and
|
||||
true is refused.
|
||||
de: |
|
||||
Ob ein Reasoning-Modell seine Denkphase durchläuft: true oder
|
||||
false; leer lässt die Voreinstellung des Servers. Ollama lässt
|
||||
Modelle wie qwen3 standardmäßig denken, ein Aufruf dauert
|
||||
dadurch rund 50 statt 10 Sekunden. Ollama bekommt think, vLLM
|
||||
chat_template_kwargs.enable_thinking. Das OpenAI-Format kennt
|
||||
keinen solchen Schalter, dort wird ein Wert abgelehnt; bei
|
||||
Anthropic wird false angenommen (erweitertes Denken ist ohne
|
||||
Anforderung aus) und true abgelehnt.
|
||||
system_prompt:
|
||||
type: text
|
||||
description:
|
||||
|
|
@ -171,6 +192,32 @@ outputs:
|
|||
keine Digest-API bereit). Zusammen mit model_endpoint und
|
||||
model_name beantwortet das die Audit-Frage "welches genaue
|
||||
Modell hat diese Antwort erzeugt?"
|
||||
tokens_prompt:
|
||||
type: text
|
||||
description:
|
||||
en: |
|
||||
How many tokens the prompt cost, as the backend counted it.
|
||||
Empty when the answer carried no count: a zero would read
|
||||
like a measurement nobody took. The hub lifts this field
|
||||
into the audit log automatically.
|
||||
de: |
|
||||
Wie viele Token die Anfrage gekostet hat, nach der Zählung
|
||||
des Backends. Leer, wenn die Antwort keine Zählung enthielt:
|
||||
eine Null würde wie eine Messung aussehen, die niemand
|
||||
gemacht hat. Der Hub hebt dieses Feld selbst in den Prüfpfad.
|
||||
tokens_completion:
|
||||
type: text
|
||||
description:
|
||||
en: |
|
||||
How many tokens the answer cost, same rules as
|
||||
tokens_prompt. Ollama counts in `eval_count`, the OpenAI
|
||||
shape in `usage.completion_tokens`, Anthropic in
|
||||
`usage.output_tokens`.
|
||||
de: |
|
||||
Wie viele Token die Antwort gekostet hat, gleiche Regeln wie
|
||||
tokens_prompt. Ollama zählt in `eval_count`, das
|
||||
OpenAI-Format in `usage.completion_tokens`, Anthropic in
|
||||
`usage.output_tokens`.
|
||||
|
||||
# Permissions required.
|
||||
#
|
||||
|
|
|
|||
17
src/lib.rs
17
src/lib.rs
|
|
@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
.get("keep_alive")
|
||||
.and_then(payload_text)
|
||||
.unwrap_or_default();
|
||||
// A flow may write `think: false` (JSON boolean) or "false".
|
||||
let think = match inputs.get("think") {
|
||||
Some(Payload::Text(s)) | Some(Payload::Json(s)) => s.clone(),
|
||||
_ => String::new(),
|
||||
};
|
||||
let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default();
|
||||
let temperature = sampling_input("temperature");
|
||||
let top_p = sampling_input("top_p");
|
||||
|
|
@ -66,6 +71,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
system_prompt: &system_prompt,
|
||||
prompt: &prompt,
|
||||
keep_alive: &keep_alive,
|
||||
think: &think,
|
||||
sampling: crate::llm::Sampling {
|
||||
temperature: &temperature,
|
||||
top_p: &top_p,
|
||||
|
|
@ -76,11 +82,20 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
||||
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
||||
|
||||
// The two token counts are the cost of this call, as the backend
|
||||
// counted it. They stay empty where the answer carried no count,
|
||||
// because an invented zero would look like a measurement.
|
||||
let (tokens_prompt, tokens_completion) = match result.tokens {
|
||||
Some((prompt, completion)) => (prompt.to_string(), completion.to_string()),
|
||||
None => (String::new(), String::new()),
|
||||
};
|
||||
Ok(Outputs::new()
|
||||
.with_text("response", result.response)
|
||||
.with_text("model_endpoint", endpoint)
|
||||
.with_text("model_name", model)
|
||||
.with_text("model_digest", result.model_digest.unwrap_or_default()))
|
||||
.with_text("model_digest", result.model_digest.unwrap_or_default())
|
||||
.with_text("tokens_prompt", tokens_prompt)
|
||||
.with_text("tokens_completion", tokens_completion))
|
||||
}
|
||||
|
||||
fn payload_text(p: &Payload) -> Option<String> {
|
||||
|
|
|
|||
247
src/llm.rs
247
src/llm.rs
|
|
@ -97,6 +97,11 @@ pub struct ChatParams<'a> {
|
|||
/// the server default, which unloads after five minutes and
|
||||
/// makes the next first request pay a 30 to 40 second reload.
|
||||
pub keep_alive: &'a str,
|
||||
/// Whether a reasoning model runs its thinking phase: "true",
|
||||
/// "false", or empty for the server default. Ollama lets
|
||||
/// qwen3-type models think by default, which takes a call from
|
||||
/// about 10 to about 50 seconds.
|
||||
pub think: &'a str,
|
||||
/// Sampling controls. Empty means "leave the server default".
|
||||
///
|
||||
/// `seed` and `temperature: 0` are what make a run repeatable,
|
||||
|
|
@ -204,6 +209,8 @@ struct OllamaRequest<'a> {
|
|||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
keep_alive: Option<KeepAlive<'a>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
think: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
options: Option<OllamaOptions>,
|
||||
}
|
||||
|
||||
|
|
@ -240,6 +247,27 @@ struct OpenAiRequest<'a> {
|
|||
top_p: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
seed: Option<SamplingValue>,
|
||||
/// vLLM only: passed to the chat template, where qwen3-type
|
||||
/// templates read `enable_thinking`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
chat_template_kwargs: Option<ChatTemplateKwargs>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct ChatTemplateKwargs {
|
||||
enable_thinking: bool,
|
||||
}
|
||||
|
||||
/// Parse the `think` input: empty is "server default".
|
||||
pub fn parse_think(raw: &str) -> Result<Option<bool>, LlmError> {
|
||||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
"" => Ok(None),
|
||||
"true" => Ok(Some(true)),
|
||||
"false" => Ok(Some(false)),
|
||||
_ => Err(LlmError::UnsupportedApi(format!(
|
||||
"think must be true or false, got {raw:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -288,6 +316,7 @@ pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
messages: build_messages(p),
|
||||
stream: false,
|
||||
keep_alive: parse_keep_alive(p.keep_alive),
|
||||
think: parse_think(p.think)?,
|
||||
options: (!options.is_empty()).then_some(options),
|
||||
};
|
||||
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||
|
|
@ -306,6 +335,22 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
.to_string(),
|
||||
));
|
||||
}
|
||||
// vLLM hands the switch to the chat template. The OpenAI wire
|
||||
// has no switch for it; refused rather than dropped, because a
|
||||
// caller who sets it expects the call to take the time it asked
|
||||
// for.
|
||||
let think = parse_think(p.think)?;
|
||||
let chat_template_kwargs = match (p.api, think) {
|
||||
(_, None) => None,
|
||||
(Api::Vllm, Some(enable_thinking)) => Some(ChatTemplateKwargs { enable_thinking }),
|
||||
(_, Some(_)) => {
|
||||
return Err(LlmError::UnsupportedApi(
|
||||
"think has no equivalent on the OpenAI chat wire; use \
|
||||
api: ollama or vllm, or drop the input"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let req = OpenAiRequest {
|
||||
model: p.model,
|
||||
messages: build_messages(p),
|
||||
|
|
@ -313,6 +358,7 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||
chat_template_kwargs,
|
||||
};
|
||||
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||
}
|
||||
|
|
@ -331,6 +377,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
.to_string(),
|
||||
));
|
||||
}
|
||||
// Extended thinking is off unless requested, so "false" is what
|
||||
// the server does anyway. Turning it on needs a token budget this
|
||||
// module does not take yet.
|
||||
if parse_think(p.think)? == Some(true) {
|
||||
return Err(LlmError::UnsupportedApi(
|
||||
"think: true is not supported for the Anthropic Messages API \
|
||||
here (extended thinking needs a token budget); use false or \
|
||||
drop the input"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
let req = AnthropicRequest {
|
||||
model: p.model,
|
||||
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
|
|
@ -393,6 +450,41 @@ pub fn extract_anthropic_content(body: &str) -> Result<String, LlmError> {
|
|||
pub struct ChatWithIdentity {
|
||||
pub response: String,
|
||||
pub model_digest: Option<String>,
|
||||
/// How many tokens the call cost, as the backend counted them:
|
||||
/// (prompt, completion). `None` where the answer carries no
|
||||
/// count, which is honest about a number nobody measured.
|
||||
pub tokens: Option<(u64, u64)>,
|
||||
}
|
||||
|
||||
/// The token counts of one answer, per wire format.
|
||||
///
|
||||
/// Three dialects, three spellings of the same two numbers: Ollama
|
||||
/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape
|
||||
/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic
|
||||
/// in `usage.input_tokens` and `usage.output_tokens`. A missing
|
||||
/// count is `None` rather than a zero, because zero is a
|
||||
/// measurement and this is the absence of one.
|
||||
pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> {
|
||||
let v: serde_json::Value = serde_json::from_str(body).ok()?;
|
||||
let pair = match api {
|
||||
Api::Ollama => (
|
||||
v.get("prompt_eval_count").and_then(|n| n.as_u64()),
|
||||
v.get("eval_count").and_then(|n| n.as_u64()),
|
||||
),
|
||||
Api::Openai | Api::Vllm => (
|
||||
v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()),
|
||||
v.pointer("/usage/completion_tokens")
|
||||
.and_then(|n| n.as_u64()),
|
||||
),
|
||||
Api::Anthropic => (
|
||||
v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()),
|
||||
v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()),
|
||||
),
|
||||
};
|
||||
match pair {
|
||||
(None, None) => None,
|
||||
(prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one chat call in the selected wire format AND, for Ollama,
|
||||
|
|
@ -429,9 +521,11 @@ pub fn chat_with_identity<C: LlmClient>(
|
|||
Api::Ollama => probe_model_digest(client, p),
|
||||
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
||||
};
|
||||
let tokens = extract_tokens(p.api, &response_body);
|
||||
Ok(ChatWithIdentity {
|
||||
response,
|
||||
model_digest,
|
||||
tokens,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -527,6 +621,7 @@ mod tests {
|
|||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
}
|
||||
}
|
||||
|
|
@ -617,6 +712,7 @@ mod tests {
|
|||
system_prompt: "be terse",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
};
|
||||
let body = build_openai_body(&p).expect("body builds");
|
||||
|
|
@ -688,6 +784,7 @@ mod tests {
|
|||
system_prompt: "",
|
||||
prompt: "hi",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
|
|
@ -710,6 +807,7 @@ mod tests {
|
|||
system_prompt: "be terse",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
};
|
||||
let body = build_anthropic_body(&p).expect("body builds");
|
||||
|
|
@ -792,6 +890,7 @@ mod tests {
|
|||
system_prompt: "",
|
||||
prompt: "hi",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
|
|
@ -815,6 +914,47 @@ mod tests {
|
|||
assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none());
|
||||
}
|
||||
|
||||
/// Three dialects, three spellings of the same two numbers.
|
||||
#[test]
|
||||
fn every_dialect_reports_its_token_counts() {
|
||||
let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#;
|
||||
assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7)));
|
||||
|
||||
let openai = r#"{"choices":[{"message":{"content":"ja"}}],
|
||||
"usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#;
|
||||
assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7)));
|
||||
assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7)));
|
||||
|
||||
let anthropic = r#"{"content":[{"text":"ja"}],
|
||||
"usage":{"input_tokens":31,"output_tokens":7}}"#;
|
||||
assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7)));
|
||||
}
|
||||
|
||||
/// A backend that counts nothing leaves the fields empty, rather
|
||||
/// than reporting a zero nobody measured.
|
||||
#[test]
|
||||
fn an_answer_without_counts_reports_none() {
|
||||
assert_eq!(
|
||||
extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#),
|
||||
None
|
||||
);
|
||||
assert_eq!(extract_tokens(Api::Anthropic, "not json"), None);
|
||||
}
|
||||
|
||||
/// One count present and the other missing still says what was
|
||||
/// measured.
|
||||
#[test]
|
||||
fn a_half_count_is_reported_as_far_as_it_goes() {
|
||||
assert_eq!(
|
||||
extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#),
|
||||
Some((31, 0))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_show_digest_finds_top_level() {
|
||||
let body = r#"{"digest":"sha256:abc"}"#;
|
||||
|
|
@ -841,6 +981,37 @@ mod tests {
|
|||
assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef"));
|
||||
}
|
||||
|
||||
/// The counts of the answer reach the caller, not only the
|
||||
/// extractor: the cost of a call belongs in the trail.
|
||||
#[test]
|
||||
fn chat_with_identity_carries_the_token_counts() {
|
||||
let canned_chat = r#"{"message":{"content":"ja"},
|
||||
"prompt_eval_count":31,"eval_count":7}"#
|
||||
.to_string();
|
||||
let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string();
|
||||
let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]);
|
||||
let p = ChatParams {
|
||||
endpoint: "http://localhost:11434/api/chat",
|
||||
..params(Api::Ollama)
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
assert_eq!(result.tokens, Some((31, 7)));
|
||||
}
|
||||
|
||||
/// A backend that counts nothing leaves the caller with nothing
|
||||
/// to report, rather than with a zero.
|
||||
#[test]
|
||||
fn chat_with_identity_reports_no_counts_when_there_are_none() {
|
||||
let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string();
|
||||
let client = MockClient::new(vec![Ok(canned_chat)]);
|
||||
let p = ChatParams {
|
||||
endpoint: "https://api.openai.com/v1/chat/completions",
|
||||
..params(Api::Openai)
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
assert_eq!(result.tokens, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chat_with_identity_swallows_show_failure() {
|
||||
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();
|
||||
|
|
@ -916,6 +1087,7 @@ mod keep_alive_tests {
|
|||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive,
|
||||
think: "",
|
||||
sampling: Sampling::default(),
|
||||
}
|
||||
}
|
||||
|
|
@ -999,6 +1171,7 @@ mod sampling_tests {
|
|||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling,
|
||||
}
|
||||
}
|
||||
|
|
@ -1103,3 +1276,77 @@ mod sampling_tests {
|
|||
assert!(build_ollama_body(&p(Api::Ollama, s)).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod think_tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
fn build_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||
match p.api {
|
||||
Api::Ollama => build_ollama_body(p),
|
||||
Api::Openai | Api::Vllm => build_openai_body(p),
|
||||
Api::Anthropic => build_anthropic_body(p),
|
||||
}
|
||||
}
|
||||
|
||||
fn p(api: Api, think: &str) -> ChatParams<'_> {
|
||||
ChatParams {
|
||||
api,
|
||||
endpoint: "http://x",
|
||||
model: "qwen3.8",
|
||||
api_key: "",
|
||||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think,
|
||||
sampling: Sampling::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Ollama lets qwen3-type models think by default; a call then
|
||||
/// takes about 50 s instead of 10 s.
|
||||
#[test]
|
||||
fn ollama_gets_think_as_a_boolean() {
|
||||
let body = build_ollama_body(&p(Api::Ollama, "false")).unwrap();
|
||||
assert!(body.contains(r#""think":false"#), "{body}");
|
||||
let body = build_ollama_body(&p(Api::Ollama, "True")).unwrap();
|
||||
assert!(body.contains(r#""think":true"#), "{body}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_leaves_every_body_unchanged() {
|
||||
for api in [Api::Ollama, Api::Openai, Api::Vllm, Api::Anthropic] {
|
||||
let body = build_body(&p(api, "")).unwrap();
|
||||
assert!(!body.contains("think"), "{body}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vllm_hands_the_switch_to_the_chat_template() {
|
||||
let body = build_body(&p(Api::Vllm, "false")).unwrap();
|
||||
assert!(
|
||||
body.contains(r#""chat_template_kwargs":{"enable_thinking":false}"#),
|
||||
"{body}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_switch_a_backend_cannot_honour_is_refused() {
|
||||
assert!(matches!(
|
||||
build_body(&p(Api::Openai, "false")),
|
||||
Err(LlmError::UnsupportedApi(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
build_body(&p(Api::Anthropic, "true")),
|
||||
Err(LlmError::UnsupportedApi(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
build_body(&p(Api::Ollama, "maybe")),
|
||||
Err(LlmError::UnsupportedApi(_))
|
||||
));
|
||||
// Off is what Anthropic does anyway.
|
||||
let body = build_body(&p(Api::Anthropic, "false")).unwrap();
|
||||
assert!(!body.contains("think"), "{body}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue