feat: max_tokens and repeat_penalty inputs, truncated and stop_reason outputs, 0.4.0
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 1s
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 1s
A model caught in a repetition loop wrote until the flow's 300 s timeout. max_tokens bounds the answer on every wire (Ollama num_predict, OpenAI max_tokens, Anthropic max_tokens), and truncated tells the caller whether the answer stopped at that limit. repeat_penalty goes to Ollama and vLLM and is refused where the backend has no such setting. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
14c63083d7
commit
72f3037ebe
8 changed files with 361 additions and 9 deletions
|
|
@ -48,6 +48,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
let top_p = sampling_input("top_p");
|
||||
let top_k = sampling_input("top_k");
|
||||
let seed = sampling_input("seed");
|
||||
let max_tokens = sampling_input("max_tokens");
|
||||
let repeat_penalty = sampling_input("repeat_penalty");
|
||||
// Refused rather than ignored for any other wire format. The
|
||||
// whole point of the input is an expectation about the server
|
||||
// ("the model stays loaded"); silently dropping it would leave
|
||||
|
|
@ -77,6 +79,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
top_p: &top_p,
|
||||
top_k: &top_k,
|
||||
seed: &seed,
|
||||
max_tokens: &max_tokens,
|
||||
repeat_penalty: &repeat_penalty,
|
||||
},
|
||||
};
|
||||
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
||||
|
|
@ -89,8 +93,13 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
Some((prompt, completion)) => (prompt.to_string(), completion.to_string()),
|
||||
None => (String::new(), String::new()),
|
||||
};
|
||||
// Empty where the backend gave no stop reason: "false" would
|
||||
// claim the answer is complete without anyone having said so.
|
||||
let truncated = result.truncated.map(|t| t.to_string()).unwrap_or_default();
|
||||
Ok(Outputs::new()
|
||||
.with_text("response", result.response)
|
||||
.with_text("truncated", truncated)
|
||||
.with_text("stop_reason", result.stop_reason.unwrap_or_default())
|
||||
.with_text("model_endpoint", endpoint)
|
||||
.with_text("model_name", model)
|
||||
.with_text("model_digest", result.model_digest.unwrap_or_default())
|
||||
|
|
|
|||
278
src/llm.rs
278
src/llm.rs
|
|
@ -119,6 +119,12 @@ pub struct Sampling<'a> {
|
|||
pub top_p: &'a str,
|
||||
pub top_k: &'a str,
|
||||
pub seed: &'a str,
|
||||
/// Upper bound on the answer length in tokens. A model caught in
|
||||
/// a repetition loop otherwise writes until the flow's timeout.
|
||||
pub max_tokens: &'a str,
|
||||
/// Penalty on repeated tokens (Ollama `repeat_penalty`, vLLM
|
||||
/// `repetition_penalty`; 1.0 is neutral).
|
||||
pub repeat_penalty: &'a str,
|
||||
}
|
||||
|
||||
/// One parsed sampling value. Numbers only; a value that is not a
|
||||
|
|
@ -160,6 +166,10 @@ struct OllamaOptions {
|
|||
top_k: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
seed: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
num_predict: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
repeat_penalty: Option<SamplingValue>,
|
||||
}
|
||||
|
||||
impl OllamaOptions {
|
||||
|
|
@ -168,6 +178,24 @@ impl OllamaOptions {
|
|||
&& self.top_p.is_none()
|
||||
&& self.top_k.is_none()
|
||||
&& self.seed.is_none()
|
||||
&& self.num_predict.is_none()
|
||||
&& self.repeat_penalty.is_none()
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `max_tokens`: a whole number of at least 1. Zero or a
|
||||
/// negative count would mean "no answer" or, for Ollama, "no limit",
|
||||
/// and neither is what a caller writing a limit asks for.
|
||||
fn parse_max_tokens(raw: &str) -> Result<Option<u32>, LlmError> {
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
match raw.parse::<u32>() {
|
||||
Ok(n) if n >= 1 => Ok(Some(n)),
|
||||
_ => Err(LlmError::UnsupportedApi(format!(
|
||||
"max_tokens must be a whole number of at least 1, got {raw:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -247,6 +275,11 @@ struct OpenAiRequest<'a> {
|
|||
top_p: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
seed: Option<SamplingValue>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
max_tokens: Option<u32>,
|
||||
/// vLLM only: an extra sampling parameter of vLLM's server.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
repetition_penalty: Option<SamplingValue>,
|
||||
/// vLLM only: passed to the chat template, where qwen3-type
|
||||
/// templates read `enable_thinking`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
|
|
@ -310,6 +343,9 @@ pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
|
||||
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||
num_predict: parse_max_tokens(p.sampling.max_tokens)?
|
||||
.map(|n| SamplingValue::Int(i64::from(n))),
|
||||
repeat_penalty: parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?,
|
||||
};
|
||||
let req = OllamaRequest {
|
||||
model: p.model,
|
||||
|
|
@ -351,6 +387,16 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
));
|
||||
}
|
||||
};
|
||||
// OpenAI's own server has no repetition penalty with this
|
||||
// meaning (frequency_penalty counts differently); vLLM has one.
|
||||
let repetition_penalty = parse_sampling("repeat_penalty", p.sampling.repeat_penalty, false)?;
|
||||
if repetition_penalty.is_some() && p.api != Api::Vllm {
|
||||
return Err(LlmError::UnsupportedApi(
|
||||
"repeat_penalty is not part of the OpenAI chat wire; use \
|
||||
api: ollama or vllm, or drop the input"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
let req = OpenAiRequest {
|
||||
model: p.model,
|
||||
messages: build_messages(p),
|
||||
|
|
@ -358,6 +404,8 @@ pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||
max_tokens: parse_max_tokens(p.sampling.max_tokens)?,
|
||||
repetition_penalty,
|
||||
chat_template_kwargs,
|
||||
};
|
||||
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||
|
|
@ -388,9 +436,17 @@ pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
|
|||
.to_string(),
|
||||
));
|
||||
}
|
||||
if !p.sampling.repeat_penalty.trim().is_empty() {
|
||||
return Err(LlmError::UnsupportedApi(
|
||||
"repeat_penalty is not part of the Anthropic Messages API; use \
|
||||
api: ollama or vllm, or drop the input"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
let req = AnthropicRequest {
|
||||
model: p.model,
|
||||
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||
max_tokens: parse_max_tokens(p.sampling.max_tokens)?
|
||||
.unwrap_or(ANTHROPIC_DEFAULT_MAX_TOKENS),
|
||||
system: if p.system_prompt.is_empty() {
|
||||
None
|
||||
} else {
|
||||
|
|
@ -454,6 +510,33 @@ pub struct ChatWithIdentity {
|
|||
/// (prompt, completion). `None` where the answer carries no
|
||||
/// count, which is honest about a number nobody measured.
|
||||
pub tokens: Option<(u64, u64)>,
|
||||
/// Why the backend stopped writing, as the backend said it
|
||||
/// (`stop`, `length`, `end_turn`, `max_tokens`, ...). `None`
|
||||
/// where the answer did not say.
|
||||
pub stop_reason: Option<String>,
|
||||
/// Whether the answer ended at the length limit rather than
|
||||
/// where the model finished. `None` where the backend gave no
|
||||
/// reason, because "not cut off" would be a guess.
|
||||
pub truncated: Option<bool>,
|
||||
}
|
||||
|
||||
/// The stop reason of one answer and whether it means "cut off at
|
||||
/// the length limit". Ollama writes `done_reason` (`stop`, `length`),
|
||||
/// the OpenAI shape `choices[0].finish_reason` (`stop`, `length`),
|
||||
/// Anthropic `stop_reason` (`end_turn`, `max_tokens`, ...).
|
||||
pub fn extract_stop(api: Api, body: &str) -> (Option<String>, Option<bool>) {
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(body) else {
|
||||
return (None, None);
|
||||
};
|
||||
let (reason, length_word) = match api {
|
||||
Api::Ollama => (v.get("done_reason"), "length"),
|
||||
Api::Openai | Api::Vllm => (v.pointer("/choices/0/finish_reason"), "length"),
|
||||
Api::Anthropic => (v.get("stop_reason"), "max_tokens"),
|
||||
};
|
||||
match reason.and_then(|r| r.as_str()).filter(|r| !r.is_empty()) {
|
||||
Some(r) => (Some(r.to_string()), Some(r == length_word)),
|
||||
None => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
/// The token counts of one answer, per wire format.
|
||||
|
|
@ -522,10 +605,13 @@ pub fn chat_with_identity<C: LlmClient>(
|
|||
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
||||
};
|
||||
let tokens = extract_tokens(p.api, &response_body);
|
||||
let (stop_reason, truncated) = extract_stop(p.api, &response_body);
|
||||
Ok(ChatWithIdentity {
|
||||
response,
|
||||
model_digest,
|
||||
tokens,
|
||||
stop_reason,
|
||||
truncated,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1350,3 +1436,193 @@ mod think_tests {
|
|||
assert!(!body.contains("think"), "{body}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod length_tests {
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> {
|
||||
ChatParams {
|
||||
api,
|
||||
endpoint: "http://x/api/chat",
|
||||
model: "qwen3.8:27b",
|
||||
api_key: "",
|
||||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
think: "",
|
||||
sampling,
|
||||
}
|
||||
}
|
||||
|
||||
fn body(api: Api, sampling: Sampling<'_>) -> Result<serde_json::Value, LlmError> {
|
||||
let params = p(api, sampling);
|
||||
let raw = match api {
|
||||
Api::Ollama => build_ollama_body(¶ms),
|
||||
Api::Openai | Api::Vllm => build_openai_body(¶ms),
|
||||
Api::Anthropic => build_anthropic_body(¶ms),
|
||||
}?;
|
||||
Ok(serde_json::from_str(&raw).unwrap())
|
||||
}
|
||||
|
||||
fn limited(max_tokens: &str) -> Sampling<'_> {
|
||||
Sampling {
|
||||
max_tokens,
|
||||
..Sampling::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Each wire spells the limit differently; every one must carry it.
|
||||
#[test]
|
||||
fn every_wire_carries_the_length_limit() {
|
||||
assert_eq!(
|
||||
body(Api::Ollama, limited("400")).unwrap()["options"]["num_predict"],
|
||||
400
|
||||
);
|
||||
assert_eq!(
|
||||
body(Api::Openai, limited("400")).unwrap()["max_tokens"],
|
||||
400
|
||||
);
|
||||
assert_eq!(body(Api::Vllm, limited("400")).unwrap()["max_tokens"], 400);
|
||||
assert_eq!(
|
||||
body(Api::Anthropic, limited("400")).unwrap()["max_tokens"],
|
||||
400
|
||||
);
|
||||
}
|
||||
|
||||
/// Unset leaves the bodies as before: no limit on Ollama and the
|
||||
/// OpenAI wire, the mandatory default on Anthropic.
|
||||
#[test]
|
||||
fn unset_keeps_the_previous_bodies() {
|
||||
let v = body(Api::Ollama, Sampling::default()).unwrap();
|
||||
assert!(v.get("options").is_none(), "{v}");
|
||||
let v = body(Api::Vllm, Sampling::default()).unwrap();
|
||||
assert!(
|
||||
v.get("max_tokens").is_none() && v.get("repetition_penalty").is_none(),
|
||||
"{v}"
|
||||
);
|
||||
assert_eq!(
|
||||
body(Api::Anthropic, Sampling::default()).unwrap()["max_tokens"],
|
||||
4096
|
||||
);
|
||||
}
|
||||
|
||||
/// Zero would be "no answer" and, for Ollama's -1, "no limit";
|
||||
/// a caller writing a limit means neither.
|
||||
#[test]
|
||||
fn a_limit_below_one_or_not_a_number_is_refused() {
|
||||
for raw in ["0", "-1", "12.5", "viel"] {
|
||||
for api in [Api::Ollama, Api::Openai, Api::Anthropic] {
|
||||
let err = body(api, limited(raw)).unwrap_err();
|
||||
assert!(
|
||||
err.to_string().contains("max_tokens"),
|
||||
"{raw} {api:?}: {err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeat_penalty_goes_where_the_backend_reads_it() {
|
||||
let s = Sampling {
|
||||
repeat_penalty: "1.15",
|
||||
..Sampling::default()
|
||||
};
|
||||
assert_eq!(
|
||||
body(Api::Ollama, s).unwrap()["options"]["repeat_penalty"],
|
||||
1.15
|
||||
);
|
||||
assert_eq!(body(Api::Vllm, s).unwrap()["repetition_penalty"], 1.15);
|
||||
// No equivalent: refused rather than dropped.
|
||||
for api in [Api::Openai, Api::Anthropic] {
|
||||
let err = body(api, s).unwrap_err();
|
||||
assert!(err.to_string().contains("repeat_penalty"), "{api:?}: {err}");
|
||||
}
|
||||
}
|
||||
|
||||
/// The case baigle hit: the answer stopped at the limit. The
|
||||
/// caller must be able to tell that from a finished answer.
|
||||
#[test]
|
||||
fn a_cut_off_answer_is_reported_per_wire() {
|
||||
let cases = [
|
||||
(
|
||||
Api::Ollama,
|
||||
r#"{"message":{"content":"§ 566"},"done_reason":"length"}"#,
|
||||
"length",
|
||||
true,
|
||||
),
|
||||
(
|
||||
Api::Ollama,
|
||||
r#"{"message":{"content":"ok"},"done_reason":"stop"}"#,
|
||||
"stop",
|
||||
false,
|
||||
),
|
||||
(
|
||||
Api::Vllm,
|
||||
r#"{"choices":[{"message":{"content":"x"},"finish_reason":"length"}]}"#,
|
||||
"length",
|
||||
true,
|
||||
),
|
||||
(
|
||||
Api::Openai,
|
||||
r#"{"choices":[{"message":{"content":"x"},"finish_reason":"stop"}]}"#,
|
||||
"stop",
|
||||
false,
|
||||
),
|
||||
(
|
||||
Api::Anthropic,
|
||||
r#"{"content":[{"text":"x"}],"stop_reason":"max_tokens"}"#,
|
||||
"max_tokens",
|
||||
true,
|
||||
),
|
||||
(
|
||||
Api::Anthropic,
|
||||
r#"{"content":[{"text":"x"}],"stop_reason":"end_turn"}"#,
|
||||
"end_turn",
|
||||
false,
|
||||
),
|
||||
];
|
||||
for (api, raw, reason, cut) in cases {
|
||||
let (r, t) = extract_stop(api, raw);
|
||||
assert_eq!(r.as_deref(), Some(reason), "{api:?}");
|
||||
assert_eq!(t, Some(cut), "{api:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// No reason in the answer is no claim either way.
|
||||
#[test]
|
||||
fn no_stop_reason_claims_nothing() {
|
||||
assert_eq!(
|
||||
extract_stop(Api::Ollama, r#"{"message":{"content":"x"}}"#),
|
||||
(None, None)
|
||||
);
|
||||
assert_eq!(extract_stop(Api::Openai, "not json"), (None, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_whole_call_reports_the_cut() {
|
||||
struct One(&'static str);
|
||||
impl LlmClient for One {
|
||||
fn post_json(
|
||||
&self,
|
||||
url: &str,
|
||||
_: &str,
|
||||
_: &[(&'static str, String)],
|
||||
) -> Result<String, LlmError> {
|
||||
if url.ends_with("/api/show") {
|
||||
Err(LlmError::Status(404))
|
||||
} else {
|
||||
Ok(self.0.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
let client = One(
|
||||
r#"{"message":{"content":"§ 556 BGB § 566 BGB"},"done_reason":"length","eval_count":400}"#,
|
||||
);
|
||||
let out = chat_with_identity(&client, &p(Api::Ollama, limited("400"))).unwrap();
|
||||
assert_eq!(out.truncated, Some(true));
|
||||
assert_eq!(out.stop_reason.as_deref(), Some("length"));
|
||||
assert_eq!(out.response, "§ 556 BGB § 566 BGB");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue