diff --git a/Cargo.lock b/Cargo.lock index e2d8ab4..7fe827e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.3.1" +version = "0.4.0" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index caceacd..9ecd984 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.3.1" +version = "0.4.0" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.de.md b/MODULE.de.md index 13b7e6c..bd68bbf 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -10,7 +10,7 @@ die Anthropic Messages API. ## Capability -- `llm.chat@0.3.0` +- `llm.chat@0.4.0` ## Eingaben @@ -22,6 +22,8 @@ die Anthropic Messages API. | `api_key` | text | Optionaler API-Key. Erscheint nie in Ausgaben, Logs oder Events. | | `keep_alive` | text | Nur Ollama: wie lange der Server das Modell nach dieser Anfrage geladen lässt (`4h`, `30m`, `-1` = unbegrenzt). Leer = Server-Voreinstellung, die nach fünf Minuten entlädt. Mit anderer `api` abgelehnt. | | `think` | text | `true` oder `false`: ob ein Reasoning-Modell seine Denkphase durchläuft. Leer = Voreinstellung des Servers (Ollama lässt Modelle wie qwen3 denken). Ollama bekommt `think`, vLLM `chat_template_kwargs.enable_thinking`; im OpenAI-Format abgelehnt; Anthropic nimmt nur `false`. Gemessen mit Ollama 0.40.9: `qwen3.8:27b` und `gemma4:26b` halten `false` ein; `qwen3:30b-a3b` denkt trotzdem, das Denken steht dann in der Antwort und endet mit ``. | +| `max_tokens` | text | Obergrenze für die Länge der Antwort in Token (ganze Zahl ab 1). Ollama bekommt `options.num_predict`, das OpenAI-Format `max_tokens`, Anthropic `max_tokens` (4096, wenn leer). Leer = keine Grenze bei Ollama und im OpenAI-Format. Ohne Grenze schreibt ein Modell in einer Wiederholungsschleife bis zur Zeitgrenze des Ablaufs; mit `num_predict: 400` brach `qwen3.8:27b` unter Ollama 0.40.9 eine solche Schleife nach 39 s ab. | +| `repeat_penalty` | text | Abschlag auf wiederholte Token, 1.0 = neutral. Ollama bekommt `options.repeat_penalty`, vLLM `repetition_penalty`; im OpenAI-Format und bei Anthropic abgelehnt. | | `system_prompt` | text | Optionale System-Nachricht. Leer = Default des Modells. | | `api` | text | Optionales Wire-Format: `ollama` (Default), `openai`, `anthropic`. | @@ -31,13 +33,15 @@ die Anthropic Messages API. | ----------- | ----------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | | `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | Verhalten wie v0.1.0, unverändert. Modell-Digest-Probe via `/api/show`. | | `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI-Chat-Completions-Format — sprechen auch vLLM und die meisten selbst gehosteten Inferenz-Server. | -| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` ist Top-Level-Feld; `max_tokens` ist Pflicht und steht fest auf 4096. | +| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` ist Top-Level-Feld; `max_tokens` ist Pflicht; `max_tokens` setzt es, sonst 4096. | ## Ausgaben | Name | Typ | Beschreibung | | ---------------- | ---- | --------------------------------------------------------------------- | | `response` | text | Die Antwort des Assistants als Plain-Text. | +| `truncated` | text | `true`, wenn die Antwort an der Längengrenze endete, `false`, wenn das Modell fertig war. Leer, wenn das Backend keinen Grund nannte. Eine abgeschnittene Antwort wird trotzdem geliefert; der Ablauf entscheidet, was damit geschieht. | +| `stop_reason` | text | Der Grund des Backends: Ollama `done_reason`, OpenAI `finish_reason`, Anthropic `stop_reason`. Leer, wenn er fehlt. | | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | | `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | diff --git a/MODULE.md b/MODULE.md index f697dd1..a1aaa32 100644 --- a/MODULE.md +++ b/MODULE.md @@ -10,7 +10,7 @@ Anthropic Messages API. ## Capability -- `llm.chat@0.3.0` +- `llm.chat@0.4.0` ## Inputs @@ -22,6 +22,8 @@ Anthropic Messages API. | `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | | `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. | | `think` | text | `true` or `false`: whether a reasoning model runs its thinking phase. Empty = server default (Ollama lets qwen3-type models think). Ollama gets `think`, vLLM `chat_template_kwargs.enable_thinking`; refused on the OpenAI wire; Anthropic takes `false` only. Measured on Ollama 0.40.9: `qwen3.8:27b` and `gemma4:26b` honour `false`; `qwen3:30b-a3b` thinks anyway and its thinking then arrives inside the reply, ending with ``. | +| `max_tokens` | text | Upper bound on the answer length in tokens (whole number, at least 1). Ollama gets `options.num_predict`, the OpenAI wire `max_tokens`, Anthropic `max_tokens` (default 4096 when empty). Empty = no limit on Ollama and the OpenAI wire. Without a limit, a model caught in a repetition loop writes until the flow's timeout; with `num_predict: 400`, `qwen3.8:27b` on Ollama 0.40.9 stopped such a loop after 39 s. | +| `repeat_penalty` | text | Penalty on repeated tokens, 1.0 = neutral. Ollama gets `options.repeat_penalty`, vLLM `repetition_penalty`; refused on the OpenAI wire and for Anthropic. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | | `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | @@ -31,13 +33,15 @@ Anthropic Messages API. | ----------- | --------------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | | `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | v0.1.0 behavior, unchanged. Model-digest probe via `/api/show`. | | `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI Chat Completions format — also spoken by vLLM and most self-hosted inference servers. | -| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` is a top-level field; `max_tokens` is mandatory and defaults to 4096. | +| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` is a top-level field; `max_tokens` is mandatory; `max_tokens` sets it, default 4096. | ## Outputs | Name | Type | Description | | ---------------- | ---- | ------------------------------------------------------------------- | | `response` | text | The assistant's plain-text reply. | +| `truncated` | text | `true` when the answer stopped at the length limit, `false` when the model finished. Empty when the backend gave no stop reason. A truncated answer is still returned; the flow decides what to do with it. | +| `stop_reason` | text | The backend's own reason: Ollama `done_reason`, OpenAI `finish_reason`, Anthropic `stop_reason`. Empty when absent. | | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | | `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | diff --git a/module.wasm b/module.wasm index c600c7f..ef73b6f 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 62ea8af..3d6b97d 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.3.1 +version: 0.4.0 # Capability provided by this module. provides: - capability: llm.chat - version: 0.3.1 + version: 0.4.0 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -108,6 +108,39 @@ inputs: das einen Lauf wiederholbar. Nur Ollama und das OpenAI-Format; Anthropic kennt keinen Seed und lehnt den Eingang ab. + max_tokens: + type: text + description: + en: | + Upper bound on the answer length in tokens, a whole number of + at least 1. A model caught in a repetition loop otherwise + writes until the flow's timeout. Ollama receives + options.num_predict, the OpenAI wire max_tokens, Anthropic + max_tokens (default 4096 there when empty). Empty leaves no + limit on Ollama and the OpenAI wire. Whether the answer + stopped at the limit is reported in the output truncated. + de: | + Obergrenze für die Länge der Antwort in Token, ganze Zahl ab + 1. Ein Modell in einer Wiederholungsschleife schreibt sonst + bis zur Zeitgrenze des Ablaufs. Ollama bekommt + options.num_predict, das OpenAI-Format max_tokens, Anthropic + max_tokens (dort 4096, wenn leer). Leer heißt bei Ollama und + im OpenAI-Format keine Grenze. Ob die Antwort an der Grenze + endete, steht in der Ausgabe truncated. + repeat_penalty: + type: text + description: + en: | + Penalty on repeated tokens, a number; 1.0 is neutral, values + such as 1.1 make loops less likely. Ollama receives + options.repeat_penalty, vLLM repetition_penalty. The OpenAI + wire and Anthropic have no such setting and refuse the input. + de: | + Abschlag auf wiederholte Token, eine Zahl; 1.0 ist neutral, + Werte wie 1.1 machen Schleifen seltener. Ollama bekommt + options.repeat_penalty, vLLM repetition_penalty. Das + OpenAI-Format und Anthropic kennen die Einstellung nicht und + lehnen den Eingang ab. keep_alive: type: text description: @@ -167,6 +200,32 @@ outputs: description: en: The assistant's plain-text reply. de: Die Klartext-Antwort des Modells. + truncated: + type: text + description: + en: | + "true" when the answer stopped at the length limit + (max_tokens) instead of where the model finished, "false" + when the model finished. Empty when the backend gave no + stop reason. + de: | + "true", wenn die Antwort an der Längengrenze (max_tokens) + endete statt dort, wo das Modell fertig war, "false", wenn + das Modell fertig war. Leer, wenn das Backend keinen Grund + für das Ende nannte. + stop_reason: + type: text + description: + en: | + Why the backend stopped writing, in its own words: Ollama + done_reason (stop, length), the OpenAI wire finish_reason + (stop, length), Anthropic stop_reason (end_turn, + max_tokens, ...). Empty when the answer did not say. + de: | + Warum das Backend aufhörte, in seinen eigenen Worten: Ollama + done_reason (stop, length), das OpenAI-Format finish_reason + (stop, length), Anthropic stop_reason (end_turn, max_tokens + u. a.). Leer, wenn die Antwort es nicht angab. model_endpoint: type: text description: diff --git a/src/lib.rs b/src/lib.rs index d4a432d..9c5749a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let top_p = sampling_input("top_p"); let top_k = sampling_input("top_k"); let seed = sampling_input("seed"); + let max_tokens = sampling_input("max_tokens"); + let repeat_penalty = sampling_input("repeat_penalty"); // Refused rather than ignored for any other wire format. The // whole point of the input is an expectation about the server // ("the model stays loaded"); silently dropping it would leave @@ -77,6 +79,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { top_p: &top_p, top_k: &top_k, seed: &seed, + max_tokens: &max_tokens, + repeat_penalty: &repeat_penalty, }, }; let result = crate::llm::chat_with_identity(&client, ¶ms) @@ -89,8 +93,13 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { Some((prompt, completion)) => (prompt.to_string(), completion.to_string()), None => (String::new(), String::new()), }; + // Empty where the backend gave no stop reason: "false" would + // claim the answer is complete without anyone having said so. + let truncated = result.truncated.map(|t| t.to_string()).unwrap_or_default(); Ok(Outputs::new() .with_text("response", result.response) + .with_text("truncated", truncated) + .with_text("stop_reason", result.stop_reason.unwrap_or_default()) .with_text("model_endpoint", endpoint) .with_text("model_name", model) .with_text("model_digest", result.model_digest.unwrap_or_default()) diff --git a/src/llm.rs b/src/llm.rs index 8fc1d6b..e5712c4 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -119,6 +119,12 @@ pub struct Sampling<'a> { pub top_p: &'a str, pub top_k: &'a str, pub seed: &'a str, + /// Upper bound on the answer length in tokens. A model caught in + /// a repetition loop otherwise writes until the flow's timeout. + pub max_tokens: &'a str, + /// Penalty on repeated tokens (Ollama `repeat_penalty`, vLLM + /// `repetition_penalty`; 1.0 is neutral). + pub repeat_penalty: &'a str, } /// One parsed sampling value. Numbers only; a value that is not a @@ -160,6 +166,10 @@ struct OllamaOptions { top_k: Option, #[serde(skip_serializing_if = "Option::is_none")] seed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + num_predict: Option, + #[serde(skip_serializing_if = "Option::is_none")] + repeat_penalty: Option, } impl OllamaOptions { @@ -168,6 +178,24 @@ impl OllamaOptions { && self.top_p.is_none() && self.top_k.is_none() && self.seed.is_none() + && self.num_predict.is_none() + && self.repeat_penalty.is_none() + } +} + +/// Parse `max_tokens`: a whole number of at least 1. Zero or a +/// negative count would mean "no answer" or, for Ollama, "no limit", +/// and neither is what a caller writing a limit asks for. +fn parse_max_tokens(raw: &str) -> Result, LlmError> { + let raw = raw.trim(); + if raw.is_empty() { + return Ok(None); + } + match raw.parse::() { + Ok(n) if n >= 1 => Ok(Some(n)), + _ => Err(LlmError::UnsupportedApi(format!( + "max_tokens must be a whole number of at least 1, got {raw:?}" + ))), } } @@ -247,6 +275,11 @@ struct OpenAiRequest<'a> { top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] seed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + max_tokens: Option, + /// vLLM only: an extra sampling parameter of vLLM's server. + #[serde(skip_serializing_if = "Option::is_none")] + repetition_penalty: Option, /// vLLM only: passed to the chat template, where qwen3-type /// templates read `enable_thinking`. #[serde(skip_serializing_if = "Option::is_none")] @@ -310,6 +343,9 @@ pub fn build_ollama_body(p: &ChatParams) -> Result { top_p: parse_sampling("top_p", p.sampling.top_p, false)?, top_k: parse_sampling("top_k", p.sampling.top_k, true)?, seed: parse_sampling("seed", p.sampling.seed, true)?, + num_predict: parse_max_tokens(p.sampling.max_tokens)? + .map(|n| SamplingValue::Int(i64::from(n))), + repeat_penalty: parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?, }; let req = OllamaRequest { model: p.model, @@ -351,6 +387,16 @@ pub fn build_openai_body(p: &ChatParams) -> Result { )); } }; + // OpenAI's own server has no repetition penalty with this + // meaning (frequency_penalty counts differently); vLLM has one. + let repetition_penalty = parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?; + if repetition_penalty.is_some() && p.api != Api::Vllm { + return Err(LlmError::UnsupportedApi( + "repeat_penalty is not part of the OpenAI chat wire; use \ + api: ollama or vllm, or drop the input" + .to_string(), + )); + } let req = OpenAiRequest { model: p.model, messages: build_messages(p), @@ -358,6 +404,8 @@ pub fn build_openai_body(p: &ChatParams) -> Result { temperature: parse_sampling("temperature", p.sampling.temperature, false)?, top_p: parse_sampling("top_p", p.sampling.top_p, false)?, seed: parse_sampling("seed", p.sampling.seed, true)?, + max_tokens: parse_max_tokens(p.sampling.max_tokens)?, + repetition_penalty, chat_template_kwargs, }; Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) @@ -388,9 +436,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result { .to_string(), )); } + if !p.sampling.repeat_penalty.trim().is_empty() { + return Err(LlmError::UnsupportedApi( + "repeat_penalty is not part of the Anthropic Messages API; use \ + api: ollama or vllm, or drop the input" + .to_string(), + )); + } let req = AnthropicRequest { model: p.model, - max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, + max_tokens: parse_max_tokens(p.sampling.max_tokens)? + .unwrap_or(ANTHROPIC_DEFAULT_MAX_TOKENS), system: if p.system_prompt.is_empty() { None } else { @@ -454,6 +510,33 @@ pub struct ChatWithIdentity { /// (prompt, completion). `None` where the answer carries no /// count, which is honest about a number nobody measured. pub tokens: Option<(u64, u64)>, + /// Why the backend stopped writing, as the backend said it + /// (`stop`, `length`, `end_turn`, `max_tokens`, ...). `None` + /// where the answer did not say. + pub stop_reason: Option, + /// Whether the answer ended at the length limit rather than + /// where the model finished. `None` where the backend gave no + /// reason, because "not cut off" would be a guess. + pub truncated: Option, +} + +/// The stop reason of one answer and whether it means "cut off at +/// the length limit". Ollama writes `done_reason` (`stop`, `length`), +/// the OpenAI shape `choices[0].finish_reason` (`stop`, `length`), +/// Anthropic `stop_reason` (`end_turn`, `max_tokens`, ...). +pub fn extract_stop(api: Api, body: &str) -> (Option, Option) { + let Ok(v) = serde_json::from_str::(body) else { + return (None, None); + }; + let (reason, length_word) = match api { + Api::Ollama => (v.get("done_reason"), "length"), + Api::Openai | Api::Vllm => (v.pointer("/choices/0/finish_reason"), "length"), + Api::Anthropic => (v.get("stop_reason"), "max_tokens"), + }; + match reason.and_then(|r| r.as_str()).filter(|r| !r.is_empty()) { + Some(r) => (Some(r.to_string()), Some(r == length_word)), + None => (None, None), + } } /// The token counts of one answer, per wire format. @@ -522,10 +605,13 @@ pub fn chat_with_identity( Api::Openai | Api::Vllm | Api::Anthropic => None, }; let tokens = extract_tokens(p.api, &response_body); + let (stop_reason, truncated) = extract_stop(p.api, &response_body); Ok(ChatWithIdentity { response, model_digest, tokens, + stop_reason, + truncated, }) } @@ -1350,3 +1436,193 @@ mod think_tests { assert!(!body.contains("think"), "{body}"); } } + +#[cfg(test)] +mod length_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + + fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> { + ChatParams { + api, + endpoint: "http://x/api/chat", + model: "qwen3.8:27b", + api_key: "", + system_prompt: "", + prompt: "hello", + keep_alive: "", + think: "", + sampling, + } + } + + fn body(api: Api, sampling: Sampling<'_>) -> Result { + let params = p(api, sampling); + let raw = match api { + Api::Ollama => build_ollama_body(¶ms), + Api::Openai | Api::Vllm => build_openai_body(¶ms), + Api::Anthropic => build_anthropic_body(¶ms), + }?; + Ok(serde_json::from_str(&raw).unwrap()) + } + + fn limited(max_tokens: &str) -> Sampling<'_> { + Sampling { + max_tokens, + ..Sampling::default() + } + } + + /// Each wire spells the limit differently; every one must carry it. + #[test] + fn every_wire_carries_the_length_limit() { + assert_eq!( + body(Api::Ollama, limited("400")).unwrap()["options"]["num_predict"], + 400 + ); + assert_eq!( + body(Api::Openai, limited("400")).unwrap()["max_tokens"], + 400 + ); + assert_eq!(body(Api::Vllm, limited("400")).unwrap()["max_tokens"], 400); + assert_eq!( + body(Api::Anthropic, limited("400")).unwrap()["max_tokens"], + 400 + ); + } + + /// Unset leaves the bodies as before: no limit on Ollama and the + /// OpenAI wire, the mandatory default on Anthropic. + #[test] + fn unset_keeps_the_previous_bodies() { + let v = body(Api::Ollama, Sampling::default()).unwrap(); + assert!(v.get("options").is_none(), "{v}"); + let v = body(Api::Vllm, Sampling::default()).unwrap(); + assert!( + v.get("max_tokens").is_none() && v.get("repetition_penalty").is_none(), + "{v}" + ); + assert_eq!( + body(Api::Anthropic, Sampling::default()).unwrap()["max_tokens"], + 4096 + ); + } + + /// Zero would be "no answer" and, for Ollama's -1, "no limit"; + /// a caller writing a limit means neither. + #[test] + fn a_limit_below_one_or_not_a_number_is_refused() { + for raw in ["0", "-1", "12.5", "viel"] { + for api in [Api::Ollama, Api::Openai, Api::Anthropic] { + let err = body(api, limited(raw)).unwrap_err(); + assert!( + err.to_string().contains("max_tokens"), + "{raw} {api:?}: {err}" + ); + } + } + } + + #[test] + fn repeat_penalty_goes_where_the_backend_reads_it() { + let s = Sampling { + repeat_penalty: "1.15", + ..Sampling::default() + }; + assert_eq!( + body(Api::Ollama, s).unwrap()["options"]["repeat_penalty"], + 1.15 + ); + assert_eq!(body(Api::Vllm, s).unwrap()["repetition_penalty"], 1.15); + // No equivalent: refused rather than dropped. + for api in [Api::Openai, Api::Anthropic] { + let err = body(api, s).unwrap_err(); + assert!(err.to_string().contains("repeat_penalty"), "{api:?}: {err}"); + } + } + + /// The case baigle hit: the answer stopped at the limit. The + /// caller must be able to tell that from a finished answer. + #[test] + fn a_cut_off_answer_is_reported_per_wire() { + let cases = [ + ( + Api::Ollama, + r#"{"message":{"content":"§ 566"},"done_reason":"length"}"#, + "length", + true, + ), + ( + Api::Ollama, + r#"{"message":{"content":"ok"},"done_reason":"stop"}"#, + "stop", + false, + ), + ( + Api::Vllm, + r#"{"choices":[{"message":{"content":"x"},"finish_reason":"length"}]}"#, + "length", + true, + ), + ( + Api::Openai, + r#"{"choices":[{"message":{"content":"x"},"finish_reason":"stop"}]}"#, + "stop", + false, + ), + ( + Api::Anthropic, + r#"{"content":[{"text":"x"}],"stop_reason":"max_tokens"}"#, + "max_tokens", + true, + ), + ( + Api::Anthropic, + r#"{"content":[{"text":"x"}],"stop_reason":"end_turn"}"#, + "end_turn", + false, + ), + ]; + for (api, raw, reason, cut) in cases { + let (r, t) = extract_stop(api, raw); + assert_eq!(r.as_deref(), Some(reason), "{api:?}"); + assert_eq!(t, Some(cut), "{api:?}"); + } + } + + /// No reason in the answer is no claim either way. + #[test] + fn no_stop_reason_claims_nothing() { + assert_eq!( + extract_stop(Api::Ollama, r#"{"message":{"content":"x"}}"#), + (None, None) + ); + assert_eq!(extract_stop(Api::Openai, "not json"), (None, None)); + } + + #[test] + fn the_whole_call_reports_the_cut() { + struct One(&'static str); + impl LlmClient for One { + fn post_json( + &self, + url: &str, + _: &str, + _: &[(&'static str, String)], + ) -> Result { + if url.ends_with("/api/show") { + Err(LlmError::Status(404)) + } else { + Ok(self.0.to_string()) + } + } + } + let client = One( + r#"{"message":{"content":"§ 556 BGB § 566 BGB"},"done_reason":"length","eval_count":400}"#, + ); + let out = chat_with_identity(&client, &p(Api::Ollama, limited("400"))).unwrap(); + assert_eq!(out.truncated, Some(true)); + assert_eq!(out.stop_reason.as_deref(), Some("length")); + assert_eq!(out.response, "§ 556 BGB § 566 BGB"); + } +}