feat: sampling options (temperature, top_p, top_k, seed)
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
aila asked for these twice (07.08. and 08.08.) and worked around it by baking the values into a derived Ollama model, which is the only way they could get a reproducible pipeline. Verified against a live Ollama: two runs with `temperature: "0"` and `seed: "42"` return the same answer. Each wire format gets what it actually supports, and an option the wire does not have is REFUSED rather than dropped: `top_k` on the OpenAI wire, `seed` on Anthropic. A silently ignored setting produces a run that looks configured and is not, which is worse than an error, especially for `seed`, where the whole point is being able to repeat the run. A non-numeric value is an error too, so a typo cannot pass as "server default". Unset leaves the request byte-identical to 0.2.1, so no existing caller sees a change. Eight guards covering all three wires, both refusals, the unset case, and that `seed` stays a whole number. Capability 0.2.1 to 0.2.2: four optional inputs, nothing removed. Signed-off-by: Stefan Flemming <stefan.a.flemming@googlemail.com>
This commit is contained in:
parent
9483f196a1
commit
a6fe962916
4 changed files with 315 additions and 21 deletions
BIN
module.wasm
BIN
module.wasm
Binary file not shown.
45
module.yaml
45
module.yaml
|
|
@ -1,12 +1,12 @@
|
||||||
schema_version: 3
|
schema_version: 3
|
||||||
provider: chain
|
provider: chain
|
||||||
name: llm-chat
|
name: llm-chat
|
||||||
version: 0.2.1
|
version: 0.2.2
|
||||||
|
|
||||||
# Capability provided by this module.
|
# Capability provided by this module.
|
||||||
provides:
|
provides:
|
||||||
- capability: llm.chat
|
- capability: llm.chat
|
||||||
version: 0.2.1
|
version: 0.2.2
|
||||||
|
|
||||||
# Declared inputs in verbose v3 form. Studio renders the
|
# Declared inputs in verbose v3 form. Studio renders the
|
||||||
# description in the active locale as a tooltip on each
|
# description in the active locale as a tooltip on each
|
||||||
|
|
@ -67,6 +67,47 @@ inputs:
|
||||||
"Authorization: Bearer", bei anthropic als "x-api-key"
|
"Authorization: Bearer", bei anthropic als "x-api-key"
|
||||||
gesendet. Leer = kein Auth-Header. Erscheint nie in
|
gesendet. Leer = kein Auth-Header. Erscheint nie in
|
||||||
Ausgaben, Logs oder Audit-Events.
|
Ausgaben, Logs oder Audit-Events.
|
||||||
|
temperature:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
Sampling temperature, e.g. "0" for the most repeatable
|
||||||
|
output. Empty leaves the server default. Works on all three
|
||||||
|
wire formats.
|
||||||
|
de: |
|
||||||
|
Sampling-Temperatur, z. B. "0" für möglichst wiederholbare
|
||||||
|
Ausgaben. Leer lässt die Server-Voreinstellung. Gilt für
|
||||||
|
alle drei Wire-Formate.
|
||||||
|
top_p:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: Nucleus sampling cutoff. Empty leaves the server default.
|
||||||
|
de: Nucleus-Sampling-Grenze. Leer lässt die Server-Voreinstellung.
|
||||||
|
top_k:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
Top-k cutoff, a whole number. Ollama and Anthropic only; with
|
||||||
|
api openai or vllm the input is refused rather than ignored,
|
||||||
|
because a setting that silently does nothing produces a run
|
||||||
|
that looks configured and is not.
|
||||||
|
de: |
|
||||||
|
Top-k-Grenze, ganze Zahl. Nur Ollama und Anthropic; mit api
|
||||||
|
openai oder vllm wird der Eingang abgelehnt statt ignoriert,
|
||||||
|
weil eine still wirkungslose Einstellung einen Lauf erzeugt,
|
||||||
|
der konfiguriert aussieht und es nicht ist.
|
||||||
|
seed:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
Sampling seed, a whole number. Together with temperature 0
|
||||||
|
this is what makes a run repeatable. Ollama and the OpenAI
|
||||||
|
wire only; Anthropic has no seed and refuses the input.
|
||||||
|
de: |
|
||||||
|
Sampling-Seed, ganze Zahl. Zusammen mit Temperatur 0 macht
|
||||||
|
das einen Lauf wiederholbar. Nur Ollama und das
|
||||||
|
OpenAI-Format; Anthropic kennt keinen Seed und lehnt den
|
||||||
|
Eingang ab.
|
||||||
keep_alive:
|
keep_alive:
|
||||||
type: text
|
type: text
|
||||||
description:
|
description:
|
||||||
|
|
|
||||||
11
src/lib.rs
11
src/lib.rs
|
|
@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
.get("keep_alive")
|
.get("keep_alive")
|
||||||
.and_then(payload_text)
|
.and_then(payload_text)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default();
|
||||||
|
let temperature = sampling_input("temperature");
|
||||||
|
let top_p = sampling_input("top_p");
|
||||||
|
let top_k = sampling_input("top_k");
|
||||||
|
let seed = sampling_input("seed");
|
||||||
// Refused rather than ignored for any other wire format. The
|
// Refused rather than ignored for any other wire format. The
|
||||||
// whole point of the input is an expectation about the server
|
// whole point of the input is an expectation about the server
|
||||||
// ("the model stays loaded"); silently dropping it would leave
|
// ("the model stays loaded"); silently dropping it would leave
|
||||||
|
|
@ -61,6 +66,12 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
system_prompt: &system_prompt,
|
system_prompt: &system_prompt,
|
||||||
prompt: &prompt,
|
prompt: &prompt,
|
||||||
keep_alive: &keep_alive,
|
keep_alive: &keep_alive,
|
||||||
|
sampling: crate::llm::Sampling {
|
||||||
|
temperature: &temperature,
|
||||||
|
top_p: &top_p,
|
||||||
|
top_k: &top_k,
|
||||||
|
seed: &seed,
|
||||||
|
},
|
||||||
};
|
};
|
||||||
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
||||||
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
||||||
|
|
|
||||||
280
src/llm.rs
280
src/llm.rs
|
|
@ -97,6 +97,73 @@ pub struct ChatParams<'a> {
|
||||||
/// the server default, which unloads after five minutes and
|
/// the server default, which unloads after five minutes and
|
||||||
/// makes the next first request pay a 30 to 40 second reload.
|
/// makes the next first request pay a 30 to 40 second reload.
|
||||||
pub keep_alive: &'a str,
|
pub keep_alive: &'a str,
|
||||||
|
/// Sampling controls. Empty means "leave the server default".
|
||||||
|
///
|
||||||
|
/// `seed` and `temperature: 0` are what make a run repeatable,
|
||||||
|
/// which matters for anything that has to be defensible later.
|
||||||
|
/// Without them, aila baked the values into a derived Ollama
|
||||||
|
/// model to get a reproducible pipeline at all.
|
||||||
|
pub sampling: Sampling<'a>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Sampling controls, as written in the flow (text, so an unset
|
||||||
|
/// input is simply empty).
|
||||||
|
#[derive(Debug, Default, Clone, Copy)]
|
||||||
|
pub struct Sampling<'a> {
|
||||||
|
pub temperature: &'a str,
|
||||||
|
pub top_p: &'a str,
|
||||||
|
pub top_k: &'a str,
|
||||||
|
pub seed: &'a str,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One parsed sampling value. Numbers only; a value that is not a
|
||||||
|
/// number is an error rather than a silently dropped setting.
|
||||||
|
#[derive(Serialize, Debug, PartialEq)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub enum SamplingValue {
|
||||||
|
Int(i64),
|
||||||
|
Float(f64),
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_sampling(name: &str, raw: &str, integer: bool) -> Result<Option<SamplingValue>, LlmError> {
|
||||||
|
let raw = raw.trim();
|
||||||
|
if raw.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if integer {
|
||||||
|
raw.parse::<i64>()
|
||||||
|
.map(|v| Some(SamplingValue::Int(v)))
|
||||||
|
.map_err(|_| {
|
||||||
|
LlmError::UnsupportedApi(format!("{name} must be a whole number, got {raw:?}"))
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
raw.parse::<f64>()
|
||||||
|
.map(|v| Some(SamplingValue::Float(v)))
|
||||||
|
.map_err(|_| LlmError::UnsupportedApi(format!("{name} must be a number, got {raw:?}")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sampling block for the Ollama wire, which nests them under
|
||||||
|
/// `options`. Omitted entirely when nothing was set.
|
||||||
|
#[derive(Serialize, Default)]
|
||||||
|
struct OllamaOptions {
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
temperature: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
top_p: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
top_k: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
seed: Option<SamplingValue>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OllamaOptions {
|
||||||
|
fn is_empty(&self) -> bool {
|
||||||
|
self.temperature.is_none()
|
||||||
|
&& self.top_p.is_none()
|
||||||
|
&& self.top_k.is_none()
|
||||||
|
&& self.seed.is_none()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Auth/protocol headers for the selected API. The api_key is only
|
/// Auth/protocol headers for the selected API. The api_key is only
|
||||||
|
|
@ -136,6 +203,8 @@ struct OllamaRequest<'a> {
|
||||||
/// version for every caller that does not use this.
|
/// version for every caller that does not use this.
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
keep_alive: Option<KeepAlive<'a>>,
|
keep_alive: Option<KeepAlive<'a>>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
options: Option<OllamaOptions>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ollama accepts either a duration string ("4h") or a number of
|
/// Ollama accepts either a duration string ("4h") or a number of
|
||||||
|
|
@ -165,6 +234,12 @@ struct OpenAiRequest<'a> {
|
||||||
model: &'a str,
|
model: &'a str,
|
||||||
messages: Vec<ChatMessage<'a>>,
|
messages: Vec<ChatMessage<'a>>,
|
||||||
stream: bool,
|
stream: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
temperature: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
top_p: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
seed: Option<SamplingValue>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
|
@ -174,6 +249,12 @@ struct AnthropicRequest<'a> {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
system: Option<&'a str>,
|
system: Option<&'a str>,
|
||||||
messages: Vec<ChatMessage<'a>>,
|
messages: Vec<ChatMessage<'a>>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
temperature: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
top_p: Option<SamplingValue>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
top_k: Option<SamplingValue>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// System-then-user message list shared by the Ollama and OpenAI
|
/// System-then-user message list shared by the Ollama and OpenAI
|
||||||
|
|
@ -195,32 +276,61 @@ fn build_messages<'a>(p: &ChatParams<'a>) -> Vec<ChatMessage<'a>> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the request body for an Ollama `/api/chat` invocation.
|
/// Build the request body for an Ollama `/api/chat` invocation.
|
||||||
pub fn build_ollama_body(p: &ChatParams) -> String {
|
pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
|
let options = OllamaOptions {
|
||||||
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||||
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||||
|
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
|
||||||
|
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||||
|
};
|
||||||
let req = OllamaRequest {
|
let req = OllamaRequest {
|
||||||
model: p.model,
|
model: p.model,
|
||||||
messages: build_messages(p),
|
messages: build_messages(p),
|
||||||
stream: false,
|
stream: false,
|
||||||
keep_alive: parse_keep_alive(p.keep_alive),
|
keep_alive: parse_keep_alive(p.keep_alive),
|
||||||
|
options: (!options.is_empty()).then_some(options),
|
||||||
};
|
};
|
||||||
serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the request body for an OpenAI-compatible
|
/// Build the request body for an OpenAI-compatible
|
||||||
/// `/v1/chat/completions` invocation (OpenAI, vLLM, most
|
/// `/v1/chat/completions` invocation (OpenAI, vLLM, most
|
||||||
/// self-hosted inference servers).
|
/// self-hosted inference servers).
|
||||||
pub fn build_openai_body(p: &ChatParams) -> String {
|
pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
|
// No top_k on this wire. Refused rather than dropped: a caller
|
||||||
|
// who set it expects it to apply.
|
||||||
|
if !p.sampling.top_k.trim().is_empty() {
|
||||||
|
return Err(LlmError::UnsupportedApi(
|
||||||
|
"top_k is not part of the OpenAI chat wire; use api: ollama \
|
||||||
|
or anthropic, or drop the input"
|
||||||
|
.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let req = OpenAiRequest {
|
let req = OpenAiRequest {
|
||||||
model: p.model,
|
model: p.model,
|
||||||
messages: build_messages(p),
|
messages: build_messages(p),
|
||||||
stream: false,
|
stream: false,
|
||||||
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||||
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||||
|
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
||||||
};
|
};
|
||||||
serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build the request body for an Anthropic `/v1/messages`
|
/// Build the request body for an Anthropic `/v1/messages`
|
||||||
/// invocation. `system` is a top-level field (not a message);
|
/// invocation. `system` is a top-level field (not a message);
|
||||||
/// `max_tokens` is mandatory in the Messages API.
|
/// `max_tokens` is mandatory in the Messages API.
|
||||||
pub fn build_anthropic_body(p: &ChatParams) -> String {
|
pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
|
||||||
|
// The Messages API has no seed. Refused rather than dropped,
|
||||||
|
// because a caller setting it is after a repeatable run and
|
||||||
|
// would otherwise believe they had one.
|
||||||
|
if !p.sampling.seed.trim().is_empty() {
|
||||||
|
return Err(LlmError::UnsupportedApi(
|
||||||
|
"seed is not part of the Anthropic Messages API; use \
|
||||||
|
api: ollama for a repeatable run, or drop the input"
|
||||||
|
.to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let req = AnthropicRequest {
|
let req = AnthropicRequest {
|
||||||
model: p.model,
|
model: p.model,
|
||||||
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
||||||
|
|
@ -233,8 +343,11 @@ pub fn build_anthropic_body(p: &ChatParams) -> String {
|
||||||
role: "user",
|
role: "user",
|
||||||
content: p.prompt,
|
content: p.prompt,
|
||||||
}],
|
}],
|
||||||
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
||||||
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
||||||
|
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
|
||||||
};
|
};
|
||||||
serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Extract the assistant's message text from an Ollama
|
/// Extract the assistant's message text from an Ollama
|
||||||
|
|
@ -304,7 +417,7 @@ pub fn chat_with_identity<C: LlmClient>(
|
||||||
Api::Ollama => build_ollama_body(p),
|
Api::Ollama => build_ollama_body(p),
|
||||||
Api::Openai | Api::Vllm => build_openai_body(p),
|
Api::Openai | Api::Vllm => build_openai_body(p),
|
||||||
Api::Anthropic => build_anthropic_body(p),
|
Api::Anthropic => build_anthropic_body(p),
|
||||||
};
|
}?;
|
||||||
let headers = build_headers(p);
|
let headers = build_headers(p);
|
||||||
let response_body = client.post_json(p.endpoint, &body, &headers)?;
|
let response_body = client.post_json(p.endpoint, &body, &headers)?;
|
||||||
let response = match p.api {
|
let response = match p.api {
|
||||||
|
|
@ -414,6 +527,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
sampling: Sampling::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -443,7 +557,7 @@ mod tests {
|
||||||
system_prompt: "be helpful",
|
system_prompt: "be helpful",
|
||||||
..params(Api::Ollama)
|
..params(Api::Ollama)
|
||||||
};
|
};
|
||||||
let body = build_ollama_body(&p);
|
let body = build_ollama_body(&p).expect("body builds");
|
||||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
let messages = v["messages"].as_array().unwrap();
|
let messages = v["messages"].as_array().unwrap();
|
||||||
assert_eq!(messages.len(), 2);
|
assert_eq!(messages.len(), 2);
|
||||||
|
|
@ -456,7 +570,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ollama_body_omits_system_when_empty() {
|
fn ollama_body_omits_system_when_empty() {
|
||||||
let body = build_ollama_body(¶ms(Api::Ollama));
|
let body = build_ollama_body(¶ms(Api::Ollama)).expect("body builds");
|
||||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
let messages = v["messages"].as_array().unwrap();
|
let messages = v["messages"].as_array().unwrap();
|
||||||
assert_eq!(messages.len(), 1);
|
assert_eq!(messages.len(), 1);
|
||||||
|
|
@ -503,8 +617,9 @@ mod tests {
|
||||||
system_prompt: "be terse",
|
system_prompt: "be terse",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let body = build_openai_body(&p);
|
let body = build_openai_body(&p).expect("body builds");
|
||||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct");
|
assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct");
|
||||||
assert_eq!(v["stream"], false);
|
assert_eq!(v["stream"], false);
|
||||||
|
|
@ -573,6 +688,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hi",
|
prompt: "hi",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let result = chat_with_identity(&client, &p).unwrap();
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
assert_eq!(result.response, "ok");
|
assert_eq!(result.response, "ok");
|
||||||
|
|
@ -594,8 +710,9 @@ mod tests {
|
||||||
system_prompt: "be terse",
|
system_prompt: "be terse",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let body = build_anthropic_body(&p);
|
let body = build_anthropic_body(&p).expect("body builds");
|
||||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
assert_eq!(v["model"], "claude-fable-5");
|
assert_eq!(v["model"], "claude-fable-5");
|
||||||
// max_tokens is mandatory in the Messages API.
|
// max_tokens is mandatory in the Messages API.
|
||||||
|
|
@ -612,7 +729,7 @@ mod tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn anthropic_body_omits_system_when_empty() {
|
fn anthropic_body_omits_system_when_empty() {
|
||||||
let body = build_anthropic_body(¶ms(Api::Anthropic));
|
let body = build_anthropic_body(¶ms(Api::Anthropic)).expect("body builds");
|
||||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
assert!(v.get("system").is_none());
|
assert!(v.get("system").is_none());
|
||||||
assert_eq!(v["max_tokens"], 4096);
|
assert_eq!(v["max_tokens"], 4096);
|
||||||
|
|
@ -675,6 +792,7 @@ mod tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hi",
|
prompt: "hi",
|
||||||
keep_alive: "",
|
keep_alive: "",
|
||||||
|
sampling: Sampling::default(),
|
||||||
};
|
};
|
||||||
let result = chat_with_identity(&client, &p).unwrap();
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
assert_eq!(result.response, "ok");
|
assert_eq!(result.response, "ok");
|
||||||
|
|
@ -786,6 +904,7 @@ mod tests {
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod keep_alive_tests {
|
mod keep_alive_tests {
|
||||||
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
fn p(api: Api, keep_alive: &str) -> ChatParams<'_> {
|
fn p(api: Api, keep_alive: &str) -> ChatParams<'_> {
|
||||||
|
|
@ -797,6 +916,7 @@ mod keep_alive_tests {
|
||||||
system_prompt: "",
|
system_prompt: "",
|
||||||
prompt: "hello",
|
prompt: "hello",
|
||||||
keep_alive,
|
keep_alive,
|
||||||
|
sampling: Sampling::default(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -805,7 +925,7 @@ mod keep_alive_tests {
|
||||||
/// second reload on the next first question.
|
/// second reload on the next first question.
|
||||||
#[test]
|
#[test]
|
||||||
fn a_duration_reaches_the_ollama_body() {
|
fn a_duration_reaches_the_ollama_body() {
|
||||||
let body = build_ollama_body(&p(Api::Ollama, "4h"));
|
let body = build_ollama_body(&p(Api::Ollama, "4h")).expect("body builds");
|
||||||
assert!(
|
assert!(
|
||||||
body.contains(r#""keep_alive":"4h""#),
|
body.contains(r#""keep_alive":"4h""#),
|
||||||
"keep_alive missing from the ollama body: {body}"
|
"keep_alive missing from the ollama body: {body}"
|
||||||
|
|
@ -818,7 +938,7 @@ mod keep_alive_tests {
|
||||||
/// prevent.
|
/// prevent.
|
||||||
#[test]
|
#[test]
|
||||||
fn minus_one_is_sent_as_a_number_not_a_string() {
|
fn minus_one_is_sent_as_a_number_not_a_string() {
|
||||||
let body = build_ollama_body(&p(Api::Ollama, "-1"));
|
let body = build_ollama_body(&p(Api::Ollama, "-1")).expect("body builds");
|
||||||
assert!(
|
assert!(
|
||||||
body.contains(r#""keep_alive":-1"#),
|
body.contains(r#""keep_alive":-1"#),
|
||||||
"expected a numeric -1, got: {body}"
|
"expected a numeric -1, got: {body}"
|
||||||
|
|
@ -831,7 +951,7 @@ mod keep_alive_tests {
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn plain_seconds_are_sent_as_a_number() {
|
fn plain_seconds_are_sent_as_a_number() {
|
||||||
let body = build_ollama_body(&p(Api::Ollama, "600"));
|
let body = build_ollama_body(&p(Api::Ollama, "600")).expect("body builds");
|
||||||
assert!(body.contains(r#""keep_alive":600"#), "{body}");
|
assert!(body.contains(r#""keep_alive":600"#), "{body}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -839,9 +959,12 @@ mod keep_alive_tests {
|
||||||
/// version, so nobody's existing flow changes behaviour.
|
/// version, so nobody's existing flow changes behaviour.
|
||||||
#[test]
|
#[test]
|
||||||
fn unset_omits_the_field_entirely() {
|
fn unset_omits_the_field_entirely() {
|
||||||
let body = build_ollama_body(&p(Api::Ollama, ""));
|
let body = build_ollama_body(&p(Api::Ollama, "")).expect("body builds");
|
||||||
assert!(!body.contains("keep_alive"), "{body}");
|
assert!(!body.contains("keep_alive"), "{body}");
|
||||||
assert_eq!(body, build_ollama_body(&p(Api::Ollama, " ")));
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
build_ollama_body(&p(Api::Ollama, " ")).expect("body builds")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The other wire formats have no such field. lib.rs refuses the
|
/// The other wire formats have no such field. lib.rs refuses the
|
||||||
|
|
@ -851,8 +974,8 @@ mod keep_alive_tests {
|
||||||
fn other_wire_formats_carry_no_keep_alive() {
|
fn other_wire_formats_carry_no_keep_alive() {
|
||||||
for api in [Api::Openai, Api::Vllm, Api::Anthropic] {
|
for api in [Api::Openai, Api::Vllm, Api::Anthropic] {
|
||||||
let body = match api {
|
let body = match api {
|
||||||
Api::Anthropic => build_anthropic_body(&p(api, "4h")),
|
Api::Anthropic => build_anthropic_body(&p(api, "4h")).expect("body builds"),
|
||||||
_ => build_openai_body(&p(api, "4h")),
|
_ => build_openai_body(&p(api, "4h")).expect("body builds"),
|
||||||
};
|
};
|
||||||
assert!(
|
assert!(
|
||||||
!body.contains("keep_alive"),
|
!body.contains("keep_alive"),
|
||||||
|
|
@ -861,3 +984,122 @@ mod keep_alive_tests {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod sampling_tests {
|
||||||
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> {
|
||||||
|
ChatParams {
|
||||||
|
api,
|
||||||
|
endpoint: "http://x",
|
||||||
|
model: "qwen",
|
||||||
|
api_key: "",
|
||||||
|
system_prompt: "",
|
||||||
|
prompt: "hello",
|
||||||
|
keep_alive: "",
|
||||||
|
sampling,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What aila needs: a repeatable run. Until now they baked the
|
||||||
|
/// values into a derived Ollama model to get one at all.
|
||||||
|
#[test]
|
||||||
|
fn ollama_nests_sampling_under_options() {
|
||||||
|
let s = Sampling {
|
||||||
|
temperature: "0",
|
||||||
|
seed: "42",
|
||||||
|
top_k: "40",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let body = build_ollama_body(&p(Api::Ollama, s)).expect("builds");
|
||||||
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(v["options"]["temperature"], 0.0);
|
||||||
|
assert_eq!(v["options"]["seed"], 42);
|
||||||
|
assert_eq!(v["options"]["top_k"], 40);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unset must leave the request exactly as it was, so no
|
||||||
|
/// existing caller sees a changed body.
|
||||||
|
#[test]
|
||||||
|
fn nothing_set_means_no_options_block() {
|
||||||
|
let body = build_ollama_body(&p(Api::Ollama, Sampling::default())).expect("builds");
|
||||||
|
assert!(!body.contains("options"), "{body}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn openai_puts_sampling_at_the_top_level() {
|
||||||
|
let s = Sampling {
|
||||||
|
temperature: "0.2",
|
||||||
|
top_p: "0.9",
|
||||||
|
seed: "7",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let body = build_openai_body(&p(Api::Openai, s)).expect("builds");
|
||||||
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(v["temperature"], 0.2);
|
||||||
|
assert_eq!(v["top_p"], 0.9);
|
||||||
|
assert_eq!(v["seed"], 7);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An option the wire does not have is refused, not dropped: a
|
||||||
|
/// caller who set it expects it to apply, and silently ignoring
|
||||||
|
/// it produces a run that looks configured and is not.
|
||||||
|
#[test]
|
||||||
|
fn openai_refuses_top_k() {
|
||||||
|
let s = Sampling {
|
||||||
|
top_k: "40",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let err = build_openai_body(&p(Api::Openai, s)).unwrap_err();
|
||||||
|
assert!(err.to_string().contains("top_k"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Anthropic has no seed, and seed is exactly what someone
|
||||||
|
/// chasing a repeatable run is after.
|
||||||
|
#[test]
|
||||||
|
fn anthropic_refuses_seed() {
|
||||||
|
let s = Sampling {
|
||||||
|
seed: "42",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let err = build_anthropic_body(&p(Api::Anthropic, s)).unwrap_err();
|
||||||
|
assert!(err.to_string().contains("seed"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn anthropic_takes_temperature_and_top_k() {
|
||||||
|
let s = Sampling {
|
||||||
|
temperature: "0.5",
|
||||||
|
top_k: "10",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let body = build_anthropic_body(&p(Api::Anthropic, s)).expect("builds");
|
||||||
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||||
|
assert_eq!(v["temperature"], 0.5);
|
||||||
|
assert_eq!(v["top_k"], 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A typo must not pass as "server default". It would look like
|
||||||
|
/// the setting applied.
|
||||||
|
#[test]
|
||||||
|
fn a_non_numeric_value_is_an_error() {
|
||||||
|
let s = Sampling {
|
||||||
|
temperature: "heiss",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
let err = build_ollama_body(&p(Api::Ollama, s)).unwrap_err();
|
||||||
|
assert!(err.to_string().contains("temperature"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `seed` must stay a whole number: 42.0 is not a seed.
|
||||||
|
#[test]
|
||||||
|
fn seed_must_be_a_whole_number() {
|
||||||
|
let s = Sampling {
|
||||||
|
seed: "42.5",
|
||||||
|
..Sampling::default()
|
||||||
|
};
|
||||||
|
assert!(build_ollama_body(&p(Api::Ollama, s)).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue