diff --git a/Cargo.lock b/Cargo.lock index 462390b..c510695 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.2.0" +version = "0.2.1" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6a761ca..6391d14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.2.0" +version = "0.2.1" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.de.md b/MODULE.de.md index ab71365..21c2d00 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -20,6 +20,7 @@ die Anthropic Messages API. | `endpoint` | text | Chat-Endpunkt-URL passend zur gewählten `api` (siehe unten). | | `model` | text | Modell-ID (z. B. `qwen2.5:14b`, `claude-fable-5`). | | `api_key` | text | Optionaler API-Key. Erscheint nie in Ausgaben, Logs oder Events. | +| `keep_alive` | text | Nur Ollama: wie lange der Server das Modell nach dieser Anfrage geladen lässt (`4h`, `30m`, `-1` = unbegrenzt). Leer = Server-Voreinstellung, die nach fünf Minuten entlädt. Mit anderer `api` abgelehnt. | | `system_prompt` | text | Optionale System-Nachricht. Leer = Default des Modells. | | `api` | text | Optionales Wire-Format: `ollama` (Default), `openai`, `anthropic`. | @@ -144,3 +145,38 @@ steps: cargo build --release --target wasm32-wasip2 # Ausgabe: target/wasm32-wasip2/release/llm_chat.wasm ``` + +## Ein lokales Modell geladen halten + +Ollama entlädt ein Modell fünf Minuten nach der letzten Anfrage. Die +nächste Anfrage zahlt dann den vollen Ladevorgang, bei einem großen +lokalen Modell 30 bis 40 Sekunden: lang genug, dass ein Bediener einen +Defekt vermutet. + +`keep_alive` setzt dieses Fenster je Anfrage: + +```yaml +- id: antwort + use: llm.chat@^0.2 + with: + endpoint: http://localhost:11434/api/chat + model: qwen2.5:7b + keep_alive: "4h" + prompt: $inputs.frage +``` + +Gegen ein laufendes Ollama gemessen (07.10.2026): ohne den Eingang lief +das Modell 5:02 nach der Anfrage ab, mit `keep_alive: "90s"` nach 92 +Sekunden. Zu beachten: Ollama behält den Wert für die geladene Instanz, +eine spätere Anfrage ohne `keep_alive` fällt also erst dann wieder auf +fünf Minuten zurück, wenn das Modell zwischendurch entladen wurde. + +`-1` wird als Zahl gesendet, nicht als Zeichenkette, weil Ollama nur +die numerische Form als „nie entladen" liest. + +Diese Einstellung gibt es nur bei Ollama. Mit `api: openai`, `vllm` +oder `anthropic` wird der Eingang **abgelehnt statt ignoriert**: er +drückt eine Erwartung an den Server aus, und ein stilles Verwerfen +würde Sie den Ladevorgang zahlen lassen, während Sie ihn für erledigt +halten. Bei vLLM und OpenAI-kompatiblen Servern wird die Verweildauer +am Server eingestellt. diff --git a/MODULE.md b/MODULE.md index cb8c167..29a511e 100644 --- a/MODULE.md +++ b/MODULE.md @@ -20,6 +20,7 @@ Anthropic Messages API. | `endpoint` | text | Chat endpoint URL matching the selected `api` (see below). | | `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | | `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | +| `keep_alive` | text | Ollama only: how long the server keeps the model resident after this request (`4h`, `30m`, `-1` = indefinitely). Empty = server default, which unloads after five minutes. Refused with any other `api`. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | | `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | @@ -140,3 +141,36 @@ steps: cargo build --release --target wasm32-wasip2 # Output: target/wasm32-wasip2/release/llm_chat.wasm ``` + +## Keeping a local model resident + +Ollama unloads a model five minutes after its last request. The next +request then pays a full reload, which is 30 to 40 seconds for a large +local model: long enough that an operator assumes something is broken. + +`keep_alive` sets that window per request: + +```yaml +- id: answer + use: llm.chat@^0.2 + with: + endpoint: http://localhost:11434/api/chat + model: qwen2.5:7b + keep_alive: "4h" + prompt: $inputs.question +``` + +Measured against a live Ollama (2026-10-07): without the input the +model expired 5:02 after the request; with `keep_alive: "90s"`, 92 +seconds after. Note that Ollama keeps the value for the loaded +instance, so a later request that omits `keep_alive` does not fall +back to five minutes until the model has actually been unloaded. + +`-1` is sent as a number, not a string, because Ollama only reads the +numeric form as "never unload". + +Only Ollama has this setting. With `api: openai`, `vllm` or +`anthropic` the input is **refused rather than ignored**: it expresses +an expectation about the server, and silently dropping it would leave +you paying the reload while believing it was handled. For vLLM and +OpenAI-compatible servers, residency is configured on the server. diff --git a/module.wasm b/module.wasm index 353c547..e1a10c5 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 03e36ae..bdfeb8c 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.2.0 +version: 0.2.1 # Capability provided by this module. provides: - capability: llm.chat - version: 0.2.0 + version: 0.2.1 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -67,6 +67,27 @@ inputs: "Authorization: Bearer", bei anthropic als "x-api-key" gesendet. Leer = kein Auth-Header. Erscheint nie in Ausgaben, Logs oder Audit-Events. + keep_alive: + type: text + description: + en: | + Ollama only: how long the server keeps the model resident + after this request, e.g. "4h", "30m", or "-1" for + indefinitely. Empty leaves the server default, which unloads + after five minutes, so the next first request pays a full + model reload (30 to 40 seconds for a large local model). + Setting this with any other api is refused rather than + ignored, because the input expresses an expectation the + other backends cannot meet. + de: | + Nur für Ollama: wie lange der Server das Modell nach dieser + Anfrage geladen lässt, z. B. "4h", "30m" oder "-1" für + unbegrenzt. Leer bedeutet Server-Voreinstellung, die nach + fünf Minuten entlädt; die nächste erste Anfrage zahlt dann + den vollen Modell-Neustart (30 bis 40 Sekunden bei einem + großen lokalen Modell). Mit einer anderen api wird der Wert + abgelehnt statt ignoriert, weil er eine Erwartung ausdrückt, + die die anderen Backends nicht erfüllen können. system_prompt: type: text description: diff --git a/src/lib.rs b/src/lib.rs index 818fa7c..bd62775 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,6 +34,24 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let api = crate::llm::Api::parse(&api_raw).map_err(|e| ModuleError::invalid_input(e.to_string()))?; + let keep_alive = inputs + .get("keep_alive") + .and_then(payload_text) + .unwrap_or_default(); + // Refused rather than ignored for any other wire format. The + // whole point of the input is an expectation about the server + // ("the model stays loaded"); silently dropping it would leave + // the caller paying a 30 to 40 second reload while believing it + // was handled. + if !keep_alive.trim().is_empty() && !matches!(api, crate::llm::Api::Ollama) { + return Err(ModuleError::invalid_input(format!( + "keep_alive is an Ollama setting and has no equivalent in the \ + {api_raw} wire format, so it cannot be honoured here. Remove \ + the input, or set api to ollama. For vLLM or OpenAI-compatible \ + servers, model residency is configured on the server." + ))); + } + let client = make_client(); let params = crate::llm::ChatParams { api, @@ -42,6 +60,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { api_key: &api_key, system_prompt: &system_prompt, prompt: &prompt, + keep_alive: &keep_alive, }; let result = crate::llm::chat_with_identity(&client, ¶ms) .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; diff --git a/src/llm.rs b/src/llm.rs index 620e29a..1214714 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -92,6 +92,11 @@ pub struct ChatParams<'a> { pub api_key: &'a str, pub system_prompt: &'a str, pub prompt: &'a str, + /// Ollama only: how long to keep the model resident after this + /// request ("4h", "30m", "-1" for indefinitely). Empty leaves + /// the server default, which unloads after five minutes and + /// makes the next first request pay a 30 to 40 second reload. + pub keep_alive: &'a str, } /// Auth/protocol headers for the selected API. The api_key is only @@ -126,6 +131,33 @@ struct OllamaRequest<'a> { model: &'a str, messages: Vec>, stream: bool, + /// Omitted entirely when unset, so the server default applies + /// and the request stays byte-identical to the previous + /// version for every caller that does not use this. + #[serde(skip_serializing_if = "Option::is_none")] + keep_alive: Option>, +} + +/// Ollama accepts either a duration string ("4h") or a number of +/// seconds, where -1 means "never unload". Sending "-1" as a string +/// is not the same thing to the server, so a value that parses as a +/// number is sent as one. +#[derive(Serialize)] +#[serde(untagged)] +enum KeepAlive<'a> { + Seconds(i64), + Duration(&'a str), +} + +fn parse_keep_alive(raw: &str) -> Option> { + let raw = raw.trim(); + if raw.is_empty() { + return None; + } + match raw.parse::() { + Ok(n) => Some(KeepAlive::Seconds(n)), + Err(_) => Some(KeepAlive::Duration(raw)), + } } #[derive(Serialize)] @@ -168,6 +200,7 @@ pub fn build_ollama_body(p: &ChatParams) -> String { model: p.model, messages: build_messages(p), stream: false, + keep_alive: parse_keep_alive(p.keep_alive), }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } @@ -380,6 +413,7 @@ mod tests { api_key: "", system_prompt: "", prompt: "hello", + keep_alive: "", } } @@ -468,6 +502,7 @@ mod tests { api_key: "sk-x", system_prompt: "be terse", prompt: "hello", + keep_alive: "", }; let body = build_openai_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); @@ -537,6 +572,7 @@ mod tests { api_key: "sk", system_prompt: "", prompt: "hi", + keep_alive: "", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -557,6 +593,7 @@ mod tests { api_key: "sk-ant", system_prompt: "be terse", prompt: "hello", + keep_alive: "", }; let body = build_anthropic_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); @@ -637,6 +674,7 @@ mod tests { api_key: "sk-ant", system_prompt: "", prompt: "hi", + keep_alive: "", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -745,3 +783,81 @@ mod tests { assert_eq!(recorded[0], recorded[1], "vllm request drifted from openai"); } } + +#[cfg(test)] +mod keep_alive_tests { + use super::*; + + fn p(api: Api, keep_alive: &str) -> ChatParams<'_> { + ChatParams { + api, + endpoint: "http://x/api/chat", + model: "qwen", + api_key: "", + system_prompt: "", + prompt: "hello", + keep_alive, + } + } + + /// The reason this input exists: Ollama unloads a model after + /// five minutes, so a demonstrator left idle pays a 30 to 40 + /// second reload on the next first question. + #[test] + fn a_duration_reaches_the_ollama_body() { + let body = build_ollama_body(&p(Api::Ollama, "4h")); + assert!( + body.contains(r#""keep_alive":"4h""#), + "keep_alive missing from the ollama body: {body}" + ); + } + + /// Ollama reads -1 as "never unload" only as a NUMBER. Quoted, + /// it is parsed as a duration string and the setting silently + /// does nothing, which is the failure this input exists to + /// prevent. + #[test] + fn minus_one_is_sent_as_a_number_not_a_string() { + let body = build_ollama_body(&p(Api::Ollama, "-1")); + assert!( + body.contains(r#""keep_alive":-1"#), + "expected a numeric -1, got: {body}" + ); + assert!( + !body.contains(r#""keep_alive":"-1""#), + "-1 was quoted, which Ollama reads as a duration: {body}" + ); + } + + #[test] + fn plain_seconds_are_sent_as_a_number() { + let body = build_ollama_body(&p(Api::Ollama, "600")); + assert!(body.contains(r#""keep_alive":600"#), "{body}"); + } + + /// Unset must leave the request byte-identical to the previous + /// version, so nobody's existing flow changes behaviour. + #[test] + fn unset_omits_the_field_entirely() { + let body = build_ollama_body(&p(Api::Ollama, "")); + assert!(!body.contains("keep_alive"), "{body}"); + assert_eq!(body, build_ollama_body(&p(Api::Ollama, " "))); + } + + /// The other wire formats have no such field. lib.rs refuses the + /// input before reaching here; this pins that the body builders + /// would not smuggle it through either. + #[test] + fn other_wire_formats_carry_no_keep_alive() { + for api in [Api::Openai, Api::Vllm, Api::Anthropic] { + let body = match api { + Api::Anthropic => build_anthropic_body(&p(api, "4h")), + _ => build_openai_body(&p(api, "4h")), + }; + assert!( + !body.contains("keep_alive"), + "{api:?} body carries keep_alive: {body}" + ); + } + } +}