feat: keep_alive input so a local model stays resident
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
Ollama unloads a model five minutes after its last request, and the next request then pays a full reload: 30 to 40 seconds for a large local model, long enough that an operator assumes something is broken. The stromnetz demonstrator runs on a notebook from around 12.10. and hits exactly that after any idle period; it works around it today by setting OLLAMA_KEEP_ALIVE on the server, which is not expressible in a flow. Measured against a live Ollama: without the input the model expired 5:02 after the request, with `keep_alive: "90s"` it expired after 92 seconds. Worth knowing for callers: Ollama keeps the value for the loaded instance, so a later request that omits keep_alive does not fall back to five minutes until the model has actually been unloaded. `-1` is sent as a number rather than a string, because Ollama reads only the numeric form as "never unload"; quoted it is parsed as a duration and the setting silently does nothing. With any other wire format the input is REFUSED, not ignored. It expresses an expectation about the server, and dropping it quietly would leave the caller paying the reload while believing it was handled. The message says what to do instead. Capability goes 0.2.0 to 0.2.1: the addition is optional and additive, so `@^0.2` and `@^0` both keep resolving, and an unset value leaves the request byte-identical to the previous version. Five guards, including that unset omits the field entirely and that the other wire formats never carry it. Mutation-checked three ways. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
dd7b419d37
commit
9483f196a1
8 changed files with 230 additions and 4 deletions
19
src/lib.rs
19
src/lib.rs
|
|
@ -34,6 +34,24 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
let api =
|
||||
crate::llm::Api::parse(&api_raw).map_err(|e| ModuleError::invalid_input(e.to_string()))?;
|
||||
|
||||
let keep_alive = inputs
|
||||
.get("keep_alive")
|
||||
.and_then(payload_text)
|
||||
.unwrap_or_default();
|
||||
// Refused rather than ignored for any other wire format. The
|
||||
// whole point of the input is an expectation about the server
|
||||
// ("the model stays loaded"); silently dropping it would leave
|
||||
// the caller paying a 30 to 40 second reload while believing it
|
||||
// was handled.
|
||||
if !keep_alive.trim().is_empty() && !matches!(api, crate::llm::Api::Ollama) {
|
||||
return Err(ModuleError::invalid_input(format!(
|
||||
"keep_alive is an Ollama setting and has no equivalent in the \
|
||||
{api_raw} wire format, so it cannot be honoured here. Remove \
|
||||
the input, or set api to ollama. For vLLM or OpenAI-compatible \
|
||||
servers, model residency is configured on the server."
|
||||
)));
|
||||
}
|
||||
|
||||
let client = make_client();
|
||||
let params = crate::llm::ChatParams {
|
||||
api,
|
||||
|
|
@ -42,6 +60,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
|||
api_key: &api_key,
|
||||
system_prompt: &system_prompt,
|
||||
prompt: &prompt,
|
||||
keep_alive: &keep_alive,
|
||||
};
|
||||
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
||||
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
||||
|
|
|
|||
116
src/llm.rs
116
src/llm.rs
|
|
@ -92,6 +92,11 @@ pub struct ChatParams<'a> {
|
|||
pub api_key: &'a str,
|
||||
pub system_prompt: &'a str,
|
||||
pub prompt: &'a str,
|
||||
/// Ollama only: how long to keep the model resident after this
|
||||
/// request ("4h", "30m", "-1" for indefinitely). Empty leaves
|
||||
/// the server default, which unloads after five minutes and
|
||||
/// makes the next first request pay a 30 to 40 second reload.
|
||||
pub keep_alive: &'a str,
|
||||
}
|
||||
|
||||
/// Auth/protocol headers for the selected API. The api_key is only
|
||||
|
|
@ -126,6 +131,33 @@ struct OllamaRequest<'a> {
|
|||
model: &'a str,
|
||||
messages: Vec<ChatMessage<'a>>,
|
||||
stream: bool,
|
||||
/// Omitted entirely when unset, so the server default applies
|
||||
/// and the request stays byte-identical to the previous
|
||||
/// version for every caller that does not use this.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
keep_alive: Option<KeepAlive<'a>>,
|
||||
}
|
||||
|
||||
/// Ollama accepts either a duration string ("4h") or a number of
|
||||
/// seconds, where -1 means "never unload". Sending "-1" as a string
|
||||
/// is not the same thing to the server, so a value that parses as a
|
||||
/// number is sent as one.
|
||||
#[derive(Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum KeepAlive<'a> {
|
||||
Seconds(i64),
|
||||
Duration(&'a str),
|
||||
}
|
||||
|
||||
fn parse_keep_alive(raw: &str) -> Option<KeepAlive<'_>> {
|
||||
let raw = raw.trim();
|
||||
if raw.is_empty() {
|
||||
return None;
|
||||
}
|
||||
match raw.parse::<i64>() {
|
||||
Ok(n) => Some(KeepAlive::Seconds(n)),
|
||||
Err(_) => Some(KeepAlive::Duration(raw)),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -168,6 +200,7 @@ pub fn build_ollama_body(p: &ChatParams) -> String {
|
|||
model: p.model,
|
||||
messages: build_messages(p),
|
||||
stream: false,
|
||||
keep_alive: parse_keep_alive(p.keep_alive),
|
||||
};
|
||||
serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
|
@ -380,6 +413,7 @@ mod tests {
|
|||
api_key: "",
|
||||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -468,6 +502,7 @@ mod tests {
|
|||
api_key: "sk-x",
|
||||
system_prompt: "be terse",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
};
|
||||
let body = build_openai_body(&p);
|
||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
|
|
@ -537,6 +572,7 @@ mod tests {
|
|||
api_key: "sk",
|
||||
system_prompt: "",
|
||||
prompt: "hi",
|
||||
keep_alive: "",
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
assert_eq!(result.response, "ok");
|
||||
|
|
@ -557,6 +593,7 @@ mod tests {
|
|||
api_key: "sk-ant",
|
||||
system_prompt: "be terse",
|
||||
prompt: "hello",
|
||||
keep_alive: "",
|
||||
};
|
||||
let body = build_anthropic_body(&p);
|
||||
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
|
|
@ -637,6 +674,7 @@ mod tests {
|
|||
api_key: "sk-ant",
|
||||
system_prompt: "",
|
||||
prompt: "hi",
|
||||
keep_alive: "",
|
||||
};
|
||||
let result = chat_with_identity(&client, &p).unwrap();
|
||||
assert_eq!(result.response, "ok");
|
||||
|
|
@ -745,3 +783,81 @@ mod tests {
|
|||
assert_eq!(recorded[0], recorded[1], "vllm request drifted from openai");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod keep_alive_tests {
|
||||
use super::*;
|
||||
|
||||
fn p(api: Api, keep_alive: &str) -> ChatParams<'_> {
|
||||
ChatParams {
|
||||
api,
|
||||
endpoint: "http://x/api/chat",
|
||||
model: "qwen",
|
||||
api_key: "",
|
||||
system_prompt: "",
|
||||
prompt: "hello",
|
||||
keep_alive,
|
||||
}
|
||||
}
|
||||
|
||||
/// The reason this input exists: Ollama unloads a model after
|
||||
/// five minutes, so a demonstrator left idle pays a 30 to 40
|
||||
/// second reload on the next first question.
|
||||
#[test]
|
||||
fn a_duration_reaches_the_ollama_body() {
|
||||
let body = build_ollama_body(&p(Api::Ollama, "4h"));
|
||||
assert!(
|
||||
body.contains(r#""keep_alive":"4h""#),
|
||||
"keep_alive missing from the ollama body: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Ollama reads -1 as "never unload" only as a NUMBER. Quoted,
|
||||
/// it is parsed as a duration string and the setting silently
|
||||
/// does nothing, which is the failure this input exists to
|
||||
/// prevent.
|
||||
#[test]
|
||||
fn minus_one_is_sent_as_a_number_not_a_string() {
|
||||
let body = build_ollama_body(&p(Api::Ollama, "-1"));
|
||||
assert!(
|
||||
body.contains(r#""keep_alive":-1"#),
|
||||
"expected a numeric -1, got: {body}"
|
||||
);
|
||||
assert!(
|
||||
!body.contains(r#""keep_alive":"-1""#),
|
||||
"-1 was quoted, which Ollama reads as a duration: {body}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plain_seconds_are_sent_as_a_number() {
|
||||
let body = build_ollama_body(&p(Api::Ollama, "600"));
|
||||
assert!(body.contains(r#""keep_alive":600"#), "{body}");
|
||||
}
|
||||
|
||||
/// Unset must leave the request byte-identical to the previous
|
||||
/// version, so nobody's existing flow changes behaviour.
|
||||
#[test]
|
||||
fn unset_omits_the_field_entirely() {
|
||||
let body = build_ollama_body(&p(Api::Ollama, ""));
|
||||
assert!(!body.contains("keep_alive"), "{body}");
|
||||
assert_eq!(body, build_ollama_body(&p(Api::Ollama, " ")));
|
||||
}
|
||||
|
||||
/// The other wire formats have no such field. lib.rs refuses the
|
||||
/// input before reaching here; this pins that the body builders
|
||||
/// would not smuggle it through either.
|
||||
#[test]
|
||||
fn other_wire_formats_carry_no_keep_alive() {
|
||||
for api in [Api::Openai, Api::Vllm, Api::Anthropic] {
|
||||
let body = match api {
|
||||
Api::Anthropic => build_anthropic_body(&p(api, "4h")),
|
||||
_ => build_openai_body(&p(api, "4h")),
|
||||
};
|
||||
assert!(
|
||||
!body.contains("keep_alive"),
|
||||
"{api:?} body carries keep_alive: {body}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue