diff --git a/module.wasm b/module.wasm index e1a10c5..b7e865b 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index bdfeb8c..43727f8 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.2.1 +version: 0.2.2 # Capability provided by this module. provides: - capability: llm.chat - version: 0.2.1 + version: 0.2.2 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -67,6 +67,47 @@ inputs: "Authorization: Bearer", bei anthropic als "x-api-key" gesendet. Leer = kein Auth-Header. Erscheint nie in Ausgaben, Logs oder Audit-Events. + temperature: + type: text + description: + en: | + Sampling temperature, e.g. "0" for the most repeatable + output. Empty leaves the server default. Works on all three + wire formats. + de: | + Sampling-Temperatur, z. B. "0" für möglichst wiederholbare + Ausgaben. Leer lässt die Server-Voreinstellung. Gilt für + alle drei Wire-Formate. + top_p: + type: text + description: + en: Nucleus sampling cutoff. Empty leaves the server default. + de: Nucleus-Sampling-Grenze. Leer lässt die Server-Voreinstellung. + top_k: + type: text + description: + en: | + Top-k cutoff, a whole number. Ollama and Anthropic only; with + api openai or vllm the input is refused rather than ignored, + because a setting that silently does nothing produces a run + that looks configured and is not. + de: | + Top-k-Grenze, ganze Zahl. Nur Ollama und Anthropic; mit api + openai oder vllm wird der Eingang abgelehnt statt ignoriert, + weil eine still wirkungslose Einstellung einen Lauf erzeugt, + der konfiguriert aussieht und es nicht ist. + seed: + type: text + description: + en: | + Sampling seed, a whole number. Together with temperature 0 + this is what makes a run repeatable. Ollama and the OpenAI + wire only; Anthropic has no seed and refuses the input. + de: | + Sampling-Seed, ganze Zahl. Zusammen mit Temperatur 0 macht + das einen Lauf wiederholbar. Nur Ollama und das + OpenAI-Format; Anthropic kennt keinen Seed und lehnt den + Eingang ab. keep_alive: type: text description: diff --git a/src/lib.rs b/src/lib.rs index bd62775..4b31f50 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,6 +38,11 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("keep_alive") .and_then(payload_text) .unwrap_or_default(); + let sampling_input = |name: &str| inputs.get(name).and_then(payload_text).unwrap_or_default(); + let temperature = sampling_input("temperature"); + let top_p = sampling_input("top_p"); + let top_k = sampling_input("top_k"); + let seed = sampling_input("seed"); // Refused rather than ignored for any other wire format. The // whole point of the input is an expectation about the server // ("the model stays loaded"); silently dropping it would leave @@ -61,6 +66,12 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { system_prompt: &system_prompt, prompt: &prompt, keep_alive: &keep_alive, + sampling: crate::llm::Sampling { + temperature: &temperature, + top_p: &top_p, + top_k: &top_k, + seed: &seed, + }, }; let result = crate::llm::chat_with_identity(&client, ¶ms) .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; diff --git a/src/llm.rs b/src/llm.rs index 1214714..bb569e4 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -97,6 +97,73 @@ pub struct ChatParams<'a> { /// the server default, which unloads after five minutes and /// makes the next first request pay a 30 to 40 second reload. pub keep_alive: &'a str, + /// Sampling controls. Empty means "leave the server default". + /// + /// `seed` and `temperature: 0` are what make a run repeatable, + /// which matters for anything that has to be defensible later. + /// Without them, aila baked the values into a derived Ollama + /// model to get a reproducible pipeline at all. + pub sampling: Sampling<'a>, +} + +/// Sampling controls, as written in the flow (text, so an unset +/// input is simply empty). +#[derive(Debug, Default, Clone, Copy)] +pub struct Sampling<'a> { + pub temperature: &'a str, + pub top_p: &'a str, + pub top_k: &'a str, + pub seed: &'a str, +} + +/// One parsed sampling value. Numbers only; a value that is not a +/// number is an error rather than a silently dropped setting. +#[derive(Serialize, Debug, PartialEq)] +#[serde(untagged)] +pub enum SamplingValue { + Int(i64), + Float(f64), +} + +fn parse_sampling(name: &str, raw: &str, integer: bool) -> Result, LlmError> { + let raw = raw.trim(); + if raw.is_empty() { + return Ok(None); + } + if integer { + raw.parse::() + .map(|v| Some(SamplingValue::Int(v))) + .map_err(|_| { + LlmError::UnsupportedApi(format!("{name} must be a whole number, got {raw:?}")) + }) + } else { + raw.parse::() + .map(|v| Some(SamplingValue::Float(v))) + .map_err(|_| LlmError::UnsupportedApi(format!("{name} must be a number, got {raw:?}"))) + } +} + +/// The sampling block for the Ollama wire, which nests them under +/// `options`. Omitted entirely when nothing was set. +#[derive(Serialize, Default)] +struct OllamaOptions { + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + top_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + seed: Option, +} + +impl OllamaOptions { + fn is_empty(&self) -> bool { + self.temperature.is_none() + && self.top_p.is_none() + && self.top_k.is_none() + && self.seed.is_none() + } } /// Auth/protocol headers for the selected API. The api_key is only @@ -136,6 +203,8 @@ struct OllamaRequest<'a> { /// version for every caller that does not use this. #[serde(skip_serializing_if = "Option::is_none")] keep_alive: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + options: Option, } /// Ollama accepts either a duration string ("4h") or a number of @@ -165,6 +234,12 @@ struct OpenAiRequest<'a> { model: &'a str, messages: Vec>, stream: bool, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + seed: Option, } #[derive(Serialize)] @@ -174,6 +249,12 @@ struct AnthropicRequest<'a> { #[serde(skip_serializing_if = "Option::is_none")] system: Option<&'a str>, messages: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + top_p: Option, + #[serde(skip_serializing_if = "Option::is_none")] + top_k: Option, } /// System-then-user message list shared by the Ollama and OpenAI @@ -195,32 +276,61 @@ fn build_messages<'a>(p: &ChatParams<'a>) -> Vec> { } /// Build the request body for an Ollama `/api/chat` invocation. -pub fn build_ollama_body(p: &ChatParams) -> String { +pub fn build_ollama_body(p: &ChatParams) -> Result { + let options = OllamaOptions { + temperature: parse_sampling("temperature", p.sampling.temperature, false)?, + top_p: parse_sampling("top_p", p.sampling.top_p, false)?, + top_k: parse_sampling("top_k", p.sampling.top_k, true)?, + seed: parse_sampling("seed", p.sampling.seed, true)?, + }; let req = OllamaRequest { model: p.model, messages: build_messages(p), stream: false, keep_alive: parse_keep_alive(p.keep_alive), + options: (!options.is_empty()).then_some(options), }; - serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) + Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } /// Build the request body for an OpenAI-compatible /// `/v1/chat/completions` invocation (OpenAI, vLLM, most /// self-hosted inference servers). -pub fn build_openai_body(p: &ChatParams) -> String { +pub fn build_openai_body(p: &ChatParams) -> Result { + // No top_k on this wire. Refused rather than dropped: a caller + // who set it expects it to apply. + if !p.sampling.top_k.trim().is_empty() { + return Err(LlmError::UnsupportedApi( + "top_k is not part of the OpenAI chat wire; use api: ollama \ + or anthropic, or drop the input" + .to_string(), + )); + } let req = OpenAiRequest { model: p.model, messages: build_messages(p), stream: false, + temperature: parse_sampling("temperature", p.sampling.temperature, false)?, + top_p: parse_sampling("top_p", p.sampling.top_p, false)?, + seed: parse_sampling("seed", p.sampling.seed, true)?, }; - serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) + Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } /// Build the request body for an Anthropic `/v1/messages` /// invocation. `system` is a top-level field (not a message); /// `max_tokens` is mandatory in the Messages API. -pub fn build_anthropic_body(p: &ChatParams) -> String { +pub fn build_anthropic_body(p: &ChatParams) -> Result { + // The Messages API has no seed. Refused rather than dropped, + // because a caller setting it is after a repeatable run and + // would otherwise believe they had one. + if !p.sampling.seed.trim().is_empty() { + return Err(LlmError::UnsupportedApi( + "seed is not part of the Anthropic Messages API; use \ + api: ollama for a repeatable run, or drop the input" + .to_string(), + )); + } let req = AnthropicRequest { model: p.model, max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, @@ -233,8 +343,11 @@ pub fn build_anthropic_body(p: &ChatParams) -> String { role: "user", content: p.prompt, }], + temperature: parse_sampling("temperature", p.sampling.temperature, false)?, + top_p: parse_sampling("top_p", p.sampling.top_p, false)?, + top_k: parse_sampling("top_k", p.sampling.top_k, true)?, }; - serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) + Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))) } /// Extract the assistant's message text from an Ollama @@ -304,7 +417,7 @@ pub fn chat_with_identity( Api::Ollama => build_ollama_body(p), Api::Openai | Api::Vllm => build_openai_body(p), Api::Anthropic => build_anthropic_body(p), - }; + }?; let headers = build_headers(p); let response_body = client.post_json(p.endpoint, &body, &headers)?; let response = match p.api { @@ -414,6 +527,7 @@ mod tests { system_prompt: "", prompt: "hello", keep_alive: "", + sampling: Sampling::default(), } } @@ -443,7 +557,7 @@ mod tests { system_prompt: "be helpful", ..params(Api::Ollama) }; - let body = build_ollama_body(&p); + let body = build_ollama_body(&p).expect("body builds"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 2); @@ -456,7 +570,7 @@ mod tests { #[test] fn ollama_body_omits_system_when_empty() { - let body = build_ollama_body(¶ms(Api::Ollama)); + let body = build_ollama_body(¶ms(Api::Ollama)).expect("body builds"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 1); @@ -503,8 +617,9 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + sampling: Sampling::default(), }; - let body = build_openai_body(&p); + let body = build_openai_body(&p).expect("body builds"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct"); assert_eq!(v["stream"], false); @@ -573,6 +688,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -594,8 +710,9 @@ mod tests { system_prompt: "be terse", prompt: "hello", keep_alive: "", + sampling: Sampling::default(), }; - let body = build_anthropic_body(&p); + let body = build_anthropic_body(&p).expect("body builds"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(v["model"], "claude-fable-5"); // max_tokens is mandatory in the Messages API. @@ -612,7 +729,7 @@ mod tests { #[test] fn anthropic_body_omits_system_when_empty() { - let body = build_anthropic_body(¶ms(Api::Anthropic)); + let body = build_anthropic_body(¶ms(Api::Anthropic)).expect("body builds"); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert!(v.get("system").is_none()); assert_eq!(v["max_tokens"], 4096); @@ -675,6 +792,7 @@ mod tests { system_prompt: "", prompt: "hi", keep_alive: "", + sampling: Sampling::default(), }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -786,6 +904,7 @@ mod tests { #[cfg(test)] mod keep_alive_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; fn p(api: Api, keep_alive: &str) -> ChatParams<'_> { @@ -797,6 +916,7 @@ mod keep_alive_tests { system_prompt: "", prompt: "hello", keep_alive, + sampling: Sampling::default(), } } @@ -805,7 +925,7 @@ mod keep_alive_tests { /// second reload on the next first question. #[test] fn a_duration_reaches_the_ollama_body() { - let body = build_ollama_body(&p(Api::Ollama, "4h")); + let body = build_ollama_body(&p(Api::Ollama, "4h")).expect("body builds"); assert!( body.contains(r#""keep_alive":"4h""#), "keep_alive missing from the ollama body: {body}" @@ -818,7 +938,7 @@ mod keep_alive_tests { /// prevent. #[test] fn minus_one_is_sent_as_a_number_not_a_string() { - let body = build_ollama_body(&p(Api::Ollama, "-1")); + let body = build_ollama_body(&p(Api::Ollama, "-1")).expect("body builds"); assert!( body.contains(r#""keep_alive":-1"#), "expected a numeric -1, got: {body}" @@ -831,7 +951,7 @@ mod keep_alive_tests { #[test] fn plain_seconds_are_sent_as_a_number() { - let body = build_ollama_body(&p(Api::Ollama, "600")); + let body = build_ollama_body(&p(Api::Ollama, "600")).expect("body builds"); assert!(body.contains(r#""keep_alive":600"#), "{body}"); } @@ -839,9 +959,12 @@ mod keep_alive_tests { /// version, so nobody's existing flow changes behaviour. #[test] fn unset_omits_the_field_entirely() { - let body = build_ollama_body(&p(Api::Ollama, "")); + let body = build_ollama_body(&p(Api::Ollama, "")).expect("body builds"); assert!(!body.contains("keep_alive"), "{body}"); - assert_eq!(body, build_ollama_body(&p(Api::Ollama, " "))); + assert_eq!( + body, + build_ollama_body(&p(Api::Ollama, " ")).expect("body builds") + ); } /// The other wire formats have no such field. lib.rs refuses the @@ -851,8 +974,8 @@ mod keep_alive_tests { fn other_wire_formats_carry_no_keep_alive() { for api in [Api::Openai, Api::Vllm, Api::Anthropic] { let body = match api { - Api::Anthropic => build_anthropic_body(&p(api, "4h")), - _ => build_openai_body(&p(api, "4h")), + Api::Anthropic => build_anthropic_body(&p(api, "4h")).expect("body builds"), + _ => build_openai_body(&p(api, "4h")).expect("body builds"), }; assert!( !body.contains("keep_alive"), @@ -861,3 +984,122 @@ mod keep_alive_tests { } } } + +#[cfg(test)] +mod sampling_tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + use super::*; + + fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> { + ChatParams { + api, + endpoint: "http://x", + model: "qwen", + api_key: "", + system_prompt: "", + prompt: "hello", + keep_alive: "", + sampling, + } + } + + /// What aila needs: a repeatable run. Until now they baked the + /// values into a derived Ollama model to get one at all. + #[test] + fn ollama_nests_sampling_under_options() { + let s = Sampling { + temperature: "0", + seed: "42", + top_k: "40", + ..Sampling::default() + }; + let body = build_ollama_body(&p(Api::Ollama, s)).expect("builds"); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["options"]["temperature"], 0.0); + assert_eq!(v["options"]["seed"], 42); + assert_eq!(v["options"]["top_k"], 40); + } + + /// Unset must leave the request exactly as it was, so no + /// existing caller sees a changed body. + #[test] + fn nothing_set_means_no_options_block() { + let body = build_ollama_body(&p(Api::Ollama, Sampling::default())).expect("builds"); + assert!(!body.contains("options"), "{body}"); + } + + #[test] + fn openai_puts_sampling_at_the_top_level() { + let s = Sampling { + temperature: "0.2", + top_p: "0.9", + seed: "7", + ..Sampling::default() + }; + let body = build_openai_body(&p(Api::Openai, s)).expect("builds"); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["temperature"], 0.2); + assert_eq!(v["top_p"], 0.9); + assert_eq!(v["seed"], 7); + } + + /// An option the wire does not have is refused, not dropped: a + /// caller who set it expects it to apply, and silently ignoring + /// it produces a run that looks configured and is not. + #[test] + fn openai_refuses_top_k() { + let s = Sampling { + top_k: "40", + ..Sampling::default() + }; + let err = build_openai_body(&p(Api::Openai, s)).unwrap_err(); + assert!(err.to_string().contains("top_k"), "{err}"); + } + + /// Anthropic has no seed, and seed is exactly what someone + /// chasing a repeatable run is after. + #[test] + fn anthropic_refuses_seed() { + let s = Sampling { + seed: "42", + ..Sampling::default() + }; + let err = build_anthropic_body(&p(Api::Anthropic, s)).unwrap_err(); + assert!(err.to_string().contains("seed"), "{err}"); + } + + #[test] + fn anthropic_takes_temperature_and_top_k() { + let s = Sampling { + temperature: "0.5", + top_k: "10", + ..Sampling::default() + }; + let body = build_anthropic_body(&p(Api::Anthropic, s)).expect("builds"); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["temperature"], 0.5); + assert_eq!(v["top_k"], 10); + } + + /// A typo must not pass as "server default". It would look like + /// the setting applied. + #[test] + fn a_non_numeric_value_is_an_error() { + let s = Sampling { + temperature: "heiss", + ..Sampling::default() + }; + let err = build_ollama_body(&p(Api::Ollama, s)).unwrap_err(); + assert!(err.to_string().contains("temperature"), "{err}"); + } + + /// `seed` must stay a whole number: 42.0 is not a seed. + #[test] + fn seed_must_be_a_whole_number() { + let s = Sampling { + seed: "42.5", + ..Sampling::default() + }; + assert!(build_ollama_body(&p(Api::Ollama, s)).is_err()); + } +}