feat: max_tokens and repeat_penalty inputs, truncated and stop_reason outputs, 0.4.0
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 1s

A model caught in a repetition loop wrote until the flow's 300 s
timeout. max_tokens bounds the answer on every wire (Ollama
num_predict, OpenAI max_tokens, Anthropic max_tokens), and
truncated tells the caller whether the answer stopped at that
limit. repeat_penalty goes to Ollama and vLLM and is refused where
the backend has no such setting.

Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
flemming-it 2026-11-12 15:49:50 +01:00
parent 14c63083d7
commit 72f3037ebe
8 changed files with 361 additions and 9 deletions

View file

@ -48,6 +48,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
let top_p = sampling_input("top_p");
let top_k = sampling_input("top_k");
let seed = sampling_input("seed");
let max_tokens = sampling_input("max_tokens");
let repeat_penalty = sampling_input("repeat_penalty");
// Refused rather than ignored for any other wire format. The
// whole point of the input is an expectation about the server
// ("the model stays loaded"); silently dropping it would leave
@ -77,6 +79,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
top_p: &top_p,
top_k: &top_k,
seed: &seed,
max_tokens: &max_tokens,
repeat_penalty: &repeat_penalty,
},
};
let result = crate::llm::chat_with_identity(&client, &params)
@ -89,8 +93,13 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
Some((prompt, completion)) => (prompt.to_string(), completion.to_string()),
None => (String::new(), String::new()),
};
// Empty where the backend gave no stop reason: "false" would
// claim the answer is complete without anyone having said so.
let truncated = result.truncated.map(|t| t.to_string()).unwrap_or_default();
Ok(Outputs::new()
.with_text("response", result.response)
.with_text("truncated", truncated)
.with_text("stop_reason", result.stop_reason.unwrap_or_default())
.with_text("model_endpoint", endpoint)
.with_text("model_name", model)
.with_text("model_digest", result.model_digest.unwrap_or_default())

View file

@ -119,6 +119,12 @@ pub struct Sampling<'a> {
pub top_p: &'a str,
pub top_k: &'a str,
pub seed: &'a str,
/// Upper bound on the answer length in tokens. A model caught in
/// a repetition loop otherwise writes until the flow's timeout.
pub max_tokens: &'a str,
/// Penalty on repeated tokens (Ollama `repeat_penalty`, vLLM
/// `repetition_penalty`; 1.0 is neutral).
pub repeat_penalty: &'a str,
}
/// One parsed sampling value. Numbers only; a value that is not a
@ -160,6 +166,10 @@ struct OllamaOptions {
top_k: Option<SamplingValue>,
#[serde(skip_serializing_if = "Option::is_none")]
seed: Option<SamplingValue>,
#[serde(skip_serializing_if = "Option::is_none")]
num_predict: Option<SamplingValue>,
#[serde(skip_serializing_if = "Option::is_none")]
repeat_penalty: Option<SamplingValue>,
}
impl OllamaOptions {
@ -168,6 +178,24 @@ impl OllamaOptions {
&& self.top_p.is_none()
&& self.top_k.is_none()
&& self.seed.is_none()
&& self.num_predict.is_none()
&& self.repeat_penalty.is_none()
}
}
/// Parse `max_tokens`: a whole number of at least 1. Zero or a
/// negative count would mean "no answer" or, for Ollama, "no limit",
/// and neither is what a caller writing a limit asks for.
fn parse_max_tokens(raw: &str) -> Result<Option<u32>, LlmError> {
let raw = raw.trim();
if raw.is_empty() {
return Ok(None);
}
match raw.parse::<u32>() {
Ok(n) if n >= 1 => Ok(Some(n)),
_ => Err(LlmError::UnsupportedApi(format!(
"max_tokens must be a whole number of at least 1, got {raw:?}"
))),
}
}
@ -247,6 +275,11 @@ struct OpenAiRequest<'a> {
top_p: Option<SamplingValue>,
#[serde(skip_serializing_if = "Option::is_none")]
seed: Option<SamplingValue>,
#[serde(skip_serializing_if = "Option::is_none")]
max_tokens: Option<u32>,
/// vLLM only: an extra sampling parameter of vLLM's server.
#[serde(skip_serializing_if = "Option::is_none")]
repetition_penalty: Option<SamplingValue>,
/// vLLM only: passed to the chat template, where qwen3-type
/// templates read `enable_thinking`.
#[serde(skip_serializing_if = "Option::is_none")]
@ -310,6 +343,9 @@ pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
seed: parse_sampling("seed", p.sampling.seed, true)?,
num_predict: parse_max_tokens(p.sampling.max_tokens)?
.map(|n| SamplingValue::Int(i64::from(n))),
repeat_penalty: parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?,
};
let req = OllamaRequest {
model: p.model,
@ -351,6 +387,16 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
));
}
};
// OpenAI's own server has no repetition penalty with this
// meaning (frequency_penalty counts differently); vLLM has one.
let repetition_penalty = parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?;
if repetition_penalty.is_some() && p.api != Api::Vllm {
return Err(LlmError::UnsupportedApi(
"repeat_penalty is not part of the OpenAI chat wire; use \
api: ollama or vllm, or drop the input"
.to_string(),
));
}
let req = OpenAiRequest {
model: p.model,
messages: build_messages(p),
@ -358,6 +404,8 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
seed: parse_sampling("seed", p.sampling.seed, true)?,
max_tokens: parse_max_tokens(p.sampling.max_tokens)?,
repetition_penalty,
chat_template_kwargs,
};
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
@ -388,9 +436,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
.to_string(),
));
}
if !p.sampling.repeat_penalty.trim().is_empty() {
return Err(LlmError::UnsupportedApi(
"repeat_penalty is not part of the Anthropic Messages API; use \
api: ollama or vllm, or drop the input"
.to_string(),
));
}
let req = AnthropicRequest {
model: p.model,
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
max_tokens: parse_max_tokens(p.sampling.max_tokens)?
.unwrap_or(ANTHROPIC_DEFAULT_MAX_TOKENS),
system: if p.system_prompt.is_empty() {
None
} else {
@ -454,6 +510,33 @@ pub struct ChatWithIdentity {
/// (prompt, completion). `None` where the answer carries no
/// count, which is honest about a number nobody measured.
pub tokens: Option<(u64, u64)>,
/// Why the backend stopped writing, as the backend said it
/// (`stop`, `length`, `end_turn`, `max_tokens`, ...). `None`
/// where the answer did not say.
pub stop_reason: Option<String>,
/// Whether the answer ended at the length limit rather than
/// where the model finished. `None` where the backend gave no
/// reason, because "not cut off" would be a guess.
pub truncated: Option<bool>,
}
/// The stop reason of one answer and whether it means "cut off at
/// the length limit". Ollama writes `done_reason` (`stop`, `length`),
/// the OpenAI shape `choices[0].finish_reason` (`stop`, `length`),
/// Anthropic `stop_reason` (`end_turn`, `max_tokens`, ...).
pub fn extract_stop(api: Api, body: &str) -> (Option<String>, Option<bool>) {
let Ok(v) = serde_json::from_str::<serde_json::Value>(body) else {
return (None, None);
};
let (reason, length_word) = match api {
Api::Ollama => (v.get("done_reason"), "length"),
Api::Openai | Api::Vllm => (v.pointer("/choices/0/finish_reason"), "length"),
Api::Anthropic => (v.get("stop_reason"), "max_tokens"),
};
match reason.and_then(|r| r.as_str()).filter(|r| !r.is_empty()) {
Some(r) => (Some(r.to_string()), Some(r == length_word)),
None => (None, None),
}
}
/// The token counts of one answer, per wire format.
@ -522,10 +605,13 @@ pub fn chat_with_identity<C: LlmClient>(
Api::Openai | Api::Vllm | Api::Anthropic => None,
};
let tokens = extract_tokens(p.api, &response_body);
let (stop_reason, truncated) = extract_stop(p.api, &response_body);
Ok(ChatWithIdentity {
response,
model_digest,
tokens,
stop_reason,
truncated,
})
}
@ -1350,3 +1436,193 @@ mod think_tests {
assert!(!body.contains("think"), "{body}");
}
}
#[cfg(test)]
mod length_tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> {
ChatParams {
api,
endpoint: "http://x/api/chat",
model: "qwen3.8:27b",
api_key: "",
system_prompt: "",
prompt: "hello",
keep_alive: "",
think: "",
sampling,
}
}
fn body(api: Api, sampling: Sampling<'_>) -> Result<serde_json::Value, LlmError> {
let params = p(api, sampling);
let raw = match api {
Api::Ollama => build_ollama_body(&params),
Api::Openai | Api::Vllm => build_openai_body(&params),
Api::Anthropic => build_anthropic_body(&params),
}?;
Ok(serde_json::from_str(&raw).unwrap())
}
fn limited(max_tokens: &str) -> Sampling<'_> {
Sampling {
max_tokens,
..Sampling::default()
}
}
/// Each wire spells the limit differently; every one must carry it.
#[test]
fn every_wire_carries_the_length_limit() {
assert_eq!(
body(Api::Ollama, limited("400")).unwrap()["options"]["num_predict"],
400
);
assert_eq!(
body(Api::Openai, limited("400")).unwrap()["max_tokens"],
400
);
assert_eq!(body(Api::Vllm, limited("400")).unwrap()["max_tokens"], 400);
assert_eq!(
body(Api::Anthropic, limited("400")).unwrap()["max_tokens"],
400
);
}
/// Unset leaves the bodies as before: no limit on Ollama and the
/// OpenAI wire, the mandatory default on Anthropic.
#[test]
fn unset_keeps_the_previous_bodies() {
let v = body(Api::Ollama, Sampling::default()).unwrap();
assert!(v.get("options").is_none(), "{v}");
let v = body(Api::Vllm, Sampling::default()).unwrap();
assert!(
v.get("max_tokens").is_none() && v.get("repetition_penalty").is_none(),
"{v}"
);
assert_eq!(
body(Api::Anthropic, Sampling::default()).unwrap()["max_tokens"],
4096
);
}
/// Zero would be "no answer" and, for Ollama's -1, "no limit";
/// a caller writing a limit means neither.
#[test]
fn a_limit_below_one_or_not_a_number_is_refused() {
for raw in ["0", "-1", "12.5", "viel"] {
for api in [Api::Ollama, Api::Openai, Api::Anthropic] {
let err = body(api, limited(raw)).unwrap_err();
assert!(
err.to_string().contains("max_tokens"),
"{raw} {api:?}: {err}"
);
}
}
}
#[test]
fn repeat_penalty_goes_where_the_backend_reads_it() {
let s = Sampling {
repeat_penalty: "1.15",
..Sampling::default()
};
assert_eq!(
body(Api::Ollama, s).unwrap()["options"]["repeat_penalty"],
1.15
);
assert_eq!(body(Api::Vllm, s).unwrap()["repetition_penalty"], 1.15);
// No equivalent: refused rather than dropped.
for api in [Api::Openai, Api::Anthropic] {
let err = body(api, s).unwrap_err();
assert!(err.to_string().contains("repeat_penalty"), "{api:?}: {err}");
}
}
/// The case baigle hit: the answer stopped at the limit. The
/// caller must be able to tell that from a finished answer.
#[test]
fn a_cut_off_answer_is_reported_per_wire() {
let cases = [
(
Api::Ollama,
r#"{"message":{"content":"§ 566"},"done_reason":"length"}"#,
"length",
true,
),
(
Api::Ollama,
r#"{"message":{"content":"ok"},"done_reason":"stop"}"#,
"stop",
false,
),
(
Api::Vllm,
r#"{"choices":[{"message":{"content":"x"},"finish_reason":"length"}]}"#,
"length",
true,
),
(
Api::Openai,
r#"{"choices":[{"message":{"content":"x"},"finish_reason":"stop"}]}"#,
"stop",
false,
),
(
Api::Anthropic,
r#"{"content":[{"text":"x"}],"stop_reason":"max_tokens"}"#,
"max_tokens",
true,
),
(
Api::Anthropic,
r#"{"content":[{"text":"x"}],"stop_reason":"end_turn"}"#,
"end_turn",
false,
),
];
for (api, raw, reason, cut) in cases {
let (r, t) = extract_stop(api, raw);
assert_eq!(r.as_deref(), Some(reason), "{api:?}");
assert_eq!(t, Some(cut), "{api:?}");
}
}
/// No reason in the answer is no claim either way.
#[test]
fn no_stop_reason_claims_nothing() {
assert_eq!(
extract_stop(Api::Ollama, r#"{"message":{"content":"x"}}"#),
(None, None)
);
assert_eq!(extract_stop(Api::Openai, "not json"), (None, None));
}
#[test]
fn the_whole_call_reports_the_cut() {
struct One(&'static str);
impl LlmClient for One {
fn post_json(
&self,
url: &str,
_: &str,
_: &[(&'static str, String)],
) -> Result<String, LlmError> {
if url.ends_with("/api/show") {
Err(LlmError::Status(404))
} else {
Ok(self.0.to_string())
}
}
}
let client = One(
r#"{"message":{"content":"§ 556 BGB § 566 BGB"},"done_reason":"length","eval_count":400}"#,
);
let out = chat_with_identity(&client, &p(Api::Ollama, limited("400"))).unwrap();
assert_eq!(out.truncated, Some(true));
assert_eq!(out.stop_reason.as_deref(), Some("length"));
assert_eq!(out.response, "§ 556 BGB § 566 BGB");
}
}