Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
aila asked for these twice (07.08. and 08.08.) and worked around it by baking the values into a derived Ollama model, which is the only way they could get a reproducible pipeline. Verified against a live Ollama: two runs with `temperature: "0"` and `seed: "42"` return the same answer. Each wire format gets what it actually supports, and an option the wire does not have is REFUSED rather than dropped: `top_k` on the OpenAI wire, `seed` on Anthropic. A silently ignored setting produces a run that looks configured and is not, which is worse than an error, especially for `seed`, where the whole point is being able to repeat the run. A non-numeric value is an error too, so a typo cannot pass as "server default". Unset leaves the request byte-identical to 0.2.1, so no existing caller sees a change. Eight guards covering all three wires, both refusals, the unset case, and that `seed` stays a whole number. Capability 0.2.1 to 0.2.2: four optional inputs, nothing removed. Signed-off-by: Stefan Flemming <stefan.a.flemming@googlemail.com>
1105 lines
38 KiB
Rust
1105 lines
38 KiB
Rust
//! Multi-API chat client: Ollama (default), OpenAI, Anthropic.
|
|
//!
|
|
//! The wire format is selected via the optional `api` input:
|
|
//!
|
|
//! - `ollama` (default) — Ollama `/api/chat`; unchanged v0.1.0
|
|
//! behavior including the best-effort model-digest probe.
|
|
//! - `openai` — OpenAI Chat Completions wire format
|
|
//! (`/v1/chat/completions`), the de-facto standard implemented
|
|
//! by vLLM and most self-hosted inference servers. Bearer auth.
|
|
//! - `anthropic` — Anthropic Messages API (`/v1/messages`),
|
|
//! `x-api-key` + `anthropic-version` headers, top-level
|
|
//! `system` field, mandatory `max_tokens`.
|
|
//!
|
|
//! No streaming, no tool calls — one prompt in, one completion out.
|
|
//!
|
|
//! All HTTP I/O lives behind a `LlmClient` trait so unit tests can
|
|
//! exercise request building, response parsing, and the digest
|
|
//! probe on the host without making real network calls.
|
|
|
|
use serde::Serialize;
|
|
|
|
/// Anthropic's Messages API requires `max_tokens`. This default is
|
|
/// large enough for document-processing completions while staying
|
|
/// well below every current model's output cap.
|
|
const ANTHROPIC_DEFAULT_MAX_TOKENS: u32 = 4096;
|
|
|
|
/// Pinned Messages API version header. Anthropic keeps old
|
|
/// versions working; bump deliberately, never implicitly.
|
|
const ANTHROPIC_VERSION: &str = "2023-06-01";
|
|
|
|
#[allow(dead_code)]
|
|
#[derive(Debug, thiserror::Error)]
|
|
pub enum LlmError {
|
|
#[error("missing required input '{0}'")]
|
|
MissingInput(&'static str),
|
|
#[error("unsupported api '{0}' (expected: ollama, openai, vllm, anthropic)")]
|
|
UnsupportedApi(String),
|
|
#[error("http error: {0}")]
|
|
Http(String),
|
|
#[error("non-success status: {0}")]
|
|
Status(u16),
|
|
#[error("response body could not be parsed as the expected schema: {0}")]
|
|
Decode(String),
|
|
}
|
|
|
|
/// Which wire format to speak. Selected by the `api` input.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum Api {
|
|
Ollama,
|
|
Openai,
|
|
/// vLLM speaks the OpenAI wire format verbatim; it is a
|
|
/// separate variant ONLY so operators can write what they
|
|
/// mean (`api: vllm`) and get vLLM-specific error hints —
|
|
/// "openai" would read as the cloud company, not a
|
|
/// self-hosted server. Same rationale as the hub's
|
|
/// SystemLlmProvider vocabulary.
|
|
Vllm,
|
|
Anthropic,
|
|
}
|
|
|
|
impl Api {
|
|
/// Parse the `api` input. Empty string means the input was not
|
|
/// provided and falls back to Ollama (v0.1.0 behavior).
|
|
pub fn parse(raw: &str) -> Result<Self, LlmError> {
|
|
match raw.trim().to_ascii_lowercase().as_str() {
|
|
"" | "ollama" => Ok(Api::Ollama),
|
|
"openai" => Ok(Api::Openai),
|
|
"vllm" => Ok(Api::Vllm),
|
|
"anthropic" => Ok(Api::Anthropic),
|
|
other => Err(LlmError::UnsupportedApi(other.to_string())),
|
|
}
|
|
}
|
|
}
|
|
|
|
pub trait LlmClient {
|
|
/// POST `body` as JSON to `url` with the given extra headers.
|
|
/// `Content-Type: application/json` is implied. Header VALUES
|
|
/// may carry credentials — implementations must never log them.
|
|
fn post_json(
|
|
&self,
|
|
url: &str,
|
|
body: &str,
|
|
headers: &[(&'static str, String)],
|
|
) -> Result<String, LlmError>;
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ChatParams<'a> {
|
|
pub api: Api,
|
|
pub endpoint: &'a str,
|
|
pub model: &'a str,
|
|
pub api_key: &'a str,
|
|
pub system_prompt: &'a str,
|
|
pub prompt: &'a str,
|
|
/// Ollama only: how long to keep the model resident after this
|
|
/// request ("4h", "30m", "-1" for indefinitely). Empty leaves
|
|
/// the server default, which unloads after five minutes and
|
|
/// makes the next first request pay a 30 to 40 second reload.
|
|
pub keep_alive: &'a str,
|
|
/// Sampling controls. Empty means "leave the server default".
|
|
///
|
|
/// `seed` and `temperature: 0` are what make a run repeatable,
|
|
/// which matters for anything that has to be defensible later.
|
|
/// Without them, aila baked the values into a derived Ollama
|
|
/// model to get a reproducible pipeline at all.
|
|
pub sampling: Sampling<'a>,
|
|
}
|
|
|
|
/// Sampling controls, as written in the flow (text, so an unset
|
|
/// input is simply empty).
|
|
#[derive(Debug, Default, Clone, Copy)]
|
|
pub struct Sampling<'a> {
|
|
pub temperature: &'a str,
|
|
pub top_p: &'a str,
|
|
pub top_k: &'a str,
|
|
pub seed: &'a str,
|
|
}
|
|
|
|
/// One parsed sampling value. Numbers only; a value that is not a
|
|
/// number is an error rather than a silently dropped setting.
|
|
#[derive(Serialize, Debug, PartialEq)]
|
|
#[serde(untagged)]
|
|
pub enum SamplingValue {
|
|
Int(i64),
|
|
Float(f64),
|
|
}
|
|
|
|
fn parse_sampling(name: &str, raw: &str, integer: bool) -> Result<Option<SamplingValue>, LlmError> {
|
|
let raw = raw.trim();
|
|
if raw.is_empty() {
|
|
return Ok(None);
|
|
}
|
|
if integer {
|
|
raw.parse::<i64>()
|
|
.map(|v| Some(SamplingValue::Int(v)))
|
|
.map_err(|_| {
|
|
LlmError::UnsupportedApi(format!("{name} must be a whole number, got {raw:?}"))
|
|
})
|
|
} else {
|
|
raw.parse::<f64>()
|
|
.map(|v| Some(SamplingValue::Float(v)))
|
|
.map_err(|_| LlmError::UnsupportedApi(format!("{name} must be a number, got {raw:?}")))
|
|
}
|
|
}
|
|
|
|
/// The sampling block for the Ollama wire, which nests them under
|
|
/// `options`. Omitted entirely when nothing was set.
|
|
#[derive(Serialize, Default)]
|
|
struct OllamaOptions {
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
temperature: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
top_p: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
top_k: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
seed: Option<SamplingValue>,
|
|
}
|
|
|
|
impl OllamaOptions {
|
|
fn is_empty(&self) -> bool {
|
|
self.temperature.is_none()
|
|
&& self.top_p.is_none()
|
|
&& self.top_k.is_none()
|
|
&& self.seed.is_none()
|
|
}
|
|
}
|
|
|
|
/// Auth/protocol headers for the selected API. The api_key is only
|
|
/// ever placed into a header value here; it must not appear in any
|
|
/// error, output, or log line.
|
|
pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> {
|
|
let mut headers = Vec::with_capacity(2);
|
|
match p.api {
|
|
Api::Ollama | Api::Openai | Api::Vllm => {
|
|
if !p.api_key.is_empty() {
|
|
headers.push(("Authorization", format!("Bearer {}", p.api_key)));
|
|
}
|
|
}
|
|
Api::Anthropic => {
|
|
if !p.api_key.is_empty() {
|
|
headers.push(("x-api-key", p.api_key.to_string()));
|
|
}
|
|
headers.push(("anthropic-version", ANTHROPIC_VERSION.to_string()));
|
|
}
|
|
}
|
|
headers
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct ChatMessage<'a> {
|
|
role: &'a str,
|
|
content: &'a str,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct OllamaRequest<'a> {
|
|
model: &'a str,
|
|
messages: Vec<ChatMessage<'a>>,
|
|
stream: bool,
|
|
/// Omitted entirely when unset, so the server default applies
|
|
/// and the request stays byte-identical to the previous
|
|
/// version for every caller that does not use this.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
keep_alive: Option<KeepAlive<'a>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
options: Option<OllamaOptions>,
|
|
}
|
|
|
|
/// Ollama accepts either a duration string ("4h") or a number of
|
|
/// seconds, where -1 means "never unload". Sending "-1" as a string
|
|
/// is not the same thing to the server, so a value that parses as a
|
|
/// number is sent as one.
|
|
#[derive(Serialize)]
|
|
#[serde(untagged)]
|
|
enum KeepAlive<'a> {
|
|
Seconds(i64),
|
|
Duration(&'a str),
|
|
}
|
|
|
|
fn parse_keep_alive(raw: &str) -> Option<KeepAlive<'_>> {
|
|
let raw = raw.trim();
|
|
if raw.is_empty() {
|
|
return None;
|
|
}
|
|
match raw.parse::<i64>() {
|
|
Ok(n) => Some(KeepAlive::Seconds(n)),
|
|
Err(_) => Some(KeepAlive::Duration(raw)),
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct OpenAiRequest<'a> {
|
|
model: &'a str,
|
|
messages: Vec<ChatMessage<'a>>,
|
|
stream: bool,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
temperature: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
top_p: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
seed: Option<SamplingValue>,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct AnthropicRequest<'a> {
|
|
model: &'a str,
|
|
max_tokens: u32,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
system: Option<&'a str>,
|
|
messages: Vec<ChatMessage<'a>>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
temperature: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
top_p: Option<SamplingValue>,
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
top_k: Option<SamplingValue>,
|
|
}
|
|
|
|
/// System-then-user message list shared by the Ollama and OpenAI
|
|
/// wire formats. The system message is omitted when empty so a
|
|
/// deployment can use the model's built-in system prompt.
|
|
fn build_messages<'a>(p: &ChatParams<'a>) -> Vec<ChatMessage<'a>> {
|
|
let mut messages = Vec::with_capacity(2);
|
|
if !p.system_prompt.is_empty() {
|
|
messages.push(ChatMessage {
|
|
role: "system",
|
|
content: p.system_prompt,
|
|
});
|
|
}
|
|
messages.push(ChatMessage {
|
|
role: "user",
|
|
content: p.prompt,
|
|
});
|
|
messages
|
|
}
|
|
|
|
/// Build the request body for an Ollama `/api/chat` invocation.
|
|
pub fn build_ollama_body(p: &ChatParams) -> Result<String, LlmError> {
|
|
let options = OllamaOptions {
|
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
|
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
|
|
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
|
};
|
|
let req = OllamaRequest {
|
|
model: p.model,
|
|
messages: build_messages(p),
|
|
stream: false,
|
|
keep_alive: parse_keep_alive(p.keep_alive),
|
|
options: (!options.is_empty()).then_some(options),
|
|
};
|
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
|
}
|
|
|
|
/// Build the request body for an OpenAI-compatible
|
|
/// `/v1/chat/completions` invocation (OpenAI, vLLM, most
|
|
/// self-hosted inference servers).
|
|
pub fn build_openai_body(p: &ChatParams) -> Result<String, LlmError> {
|
|
// No top_k on this wire. Refused rather than dropped: a caller
|
|
// who set it expects it to apply.
|
|
if !p.sampling.top_k.trim().is_empty() {
|
|
return Err(LlmError::UnsupportedApi(
|
|
"top_k is not part of the OpenAI chat wire; use api: ollama \
|
|
or anthropic, or drop the input"
|
|
.to_string(),
|
|
));
|
|
}
|
|
let req = OpenAiRequest {
|
|
model: p.model,
|
|
messages: build_messages(p),
|
|
stream: false,
|
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
|
seed: parse_sampling("seed", p.sampling.seed, true)?,
|
|
};
|
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
|
}
|
|
|
|
/// Build the request body for an Anthropic `/v1/messages`
|
|
/// invocation. `system` is a top-level field (not a message);
|
|
/// `max_tokens` is mandatory in the Messages API.
|
|
pub fn build_anthropic_body(p: &ChatParams) -> Result<String, LlmError> {
|
|
// The Messages API has no seed. Refused rather than dropped,
|
|
// because a caller setting it is after a repeatable run and
|
|
// would otherwise believe they had one.
|
|
if !p.sampling.seed.trim().is_empty() {
|
|
return Err(LlmError::UnsupportedApi(
|
|
"seed is not part of the Anthropic Messages API; use \
|
|
api: ollama for a repeatable run, or drop the input"
|
|
.to_string(),
|
|
));
|
|
}
|
|
let req = AnthropicRequest {
|
|
model: p.model,
|
|
max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS,
|
|
system: if p.system_prompt.is_empty() {
|
|
None
|
|
} else {
|
|
Some(p.system_prompt)
|
|
},
|
|
messages: vec![ChatMessage {
|
|
role: "user",
|
|
content: p.prompt,
|
|
}],
|
|
temperature: parse_sampling("temperature", p.sampling.temperature, false)?,
|
|
top_p: parse_sampling("top_p", p.sampling.top_p, false)?,
|
|
top_k: parse_sampling("top_k", p.sampling.top_k, true)?,
|
|
};
|
|
Ok(serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")))
|
|
}
|
|
|
|
/// Extract the assistant's message text from an Ollama
|
|
/// /api/chat response.
|
|
pub fn extract_ollama_content(body: &str) -> Result<String, LlmError> {
|
|
let v: serde_json::Value =
|
|
serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?;
|
|
v.get("message")
|
|
.and_then(|m| m.get("content"))
|
|
.and_then(|c| c.as_str())
|
|
.map(|s| s.to_string())
|
|
.ok_or_else(|| LlmError::Decode("missing message.content".into()))
|
|
}
|
|
|
|
/// Extract the assistant text from an OpenAI Chat Completions
|
|
/// response: `choices[0].message.content`.
|
|
pub fn extract_openai_content(body: &str) -> Result<String, LlmError> {
|
|
let v: serde_json::Value =
|
|
serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?;
|
|
v.get("choices")
|
|
.and_then(|c| c.get(0))
|
|
.and_then(|c| c.get("message"))
|
|
.and_then(|m| m.get("content"))
|
|
.and_then(|c| c.as_str())
|
|
.map(|s| s.to_string())
|
|
.ok_or_else(|| LlmError::Decode("missing choices[0].message.content".into()))
|
|
}
|
|
|
|
/// Extract the assistant text from an Anthropic Messages
|
|
/// response: `content[0].text`.
|
|
pub fn extract_anthropic_content(body: &str) -> Result<String, LlmError> {
|
|
let v: serde_json::Value =
|
|
serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?;
|
|
v.get("content")
|
|
.and_then(|c| c.get(0))
|
|
.and_then(|c| c.get("text"))
|
|
.and_then(|t| t.as_str())
|
|
.map(|s| s.to_string())
|
|
.ok_or_else(|| LlmError::Decode("missing content[0].text".into()))
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ChatWithIdentity {
|
|
pub response: String,
|
|
pub model_digest: Option<String>,
|
|
}
|
|
|
|
/// Run one chat call in the selected wire format AND, for Ollama,
|
|
/// probe the model digest. The probe is best-effort — non-Ollama
|
|
/// APIs expose no digest endpoint, so `model_digest = None` there
|
|
/// (documented empty, never invented); transient probe failures on
|
|
/// Ollama also yield `None` rather than failing the whole call.
|
|
pub fn chat_with_identity<C: LlmClient>(
|
|
client: &C,
|
|
p: &ChatParams,
|
|
) -> Result<ChatWithIdentity, LlmError> {
|
|
if p.endpoint.is_empty() {
|
|
return Err(LlmError::MissingInput("endpoint"));
|
|
}
|
|
if p.model.is_empty() {
|
|
return Err(LlmError::MissingInput("model"));
|
|
}
|
|
if p.prompt.is_empty() {
|
|
return Err(LlmError::MissingInput("prompt"));
|
|
}
|
|
let body = match p.api {
|
|
Api::Ollama => build_ollama_body(p),
|
|
Api::Openai | Api::Vllm => build_openai_body(p),
|
|
Api::Anthropic => build_anthropic_body(p),
|
|
}?;
|
|
let headers = build_headers(p);
|
|
let response_body = client.post_json(p.endpoint, &body, &headers)?;
|
|
let response = match p.api {
|
|
Api::Ollama => extract_ollama_content(&response_body)?,
|
|
Api::Openai | Api::Vllm => extract_openai_content(&response_body)?,
|
|
Api::Anthropic => extract_anthropic_content(&response_body)?,
|
|
};
|
|
let model_digest = match p.api {
|
|
Api::Ollama => probe_model_digest(client, p),
|
|
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
|
};
|
|
Ok(ChatWithIdentity {
|
|
response,
|
|
model_digest,
|
|
})
|
|
}
|
|
|
|
fn probe_model_digest<C: LlmClient>(client: &C, p: &ChatParams) -> Option<String> {
|
|
let show_url = derive_show_url(p.endpoint)?;
|
|
let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?;
|
|
let headers = build_headers(p);
|
|
let response = client.post_json(&show_url, &body, &headers).ok()?;
|
|
extract_show_digest(&response)
|
|
}
|
|
|
|
pub fn derive_show_url(chat_endpoint: &str) -> Option<String> {
|
|
if chat_endpoint.ends_with("/api/chat") {
|
|
let head_len = chat_endpoint.len() - "/api/chat".len();
|
|
let mut url = String::with_capacity(head_len + "/api/show".len());
|
|
url.push_str(&chat_endpoint[..head_len]);
|
|
url.push_str("/api/show");
|
|
Some(url)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
pub fn extract_show_digest(body: &str) -> Option<String> {
|
|
let v: serde_json::Value = serde_json::from_str(body).ok()?;
|
|
let candidates = [
|
|
v.get("digest"),
|
|
v.get("details").and_then(|d| d.get("digest")),
|
|
v.get("model_info").and_then(|d| d.get("digest")),
|
|
];
|
|
for cand in candidates {
|
|
if let Some(s) = cand.and_then(|v| v.as_str()) {
|
|
if !s.is_empty() {
|
|
return Some(s.to_string());
|
|
}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
use super::*;
|
|
use std::cell::RefCell;
|
|
|
|
/// (url, body, headers) of one recorded call.
|
|
type RecordedCall = (String, String, Vec<(String, String)>);
|
|
|
|
struct MockClient {
|
|
responses: RefCell<Vec<Result<String, LlmError>>>,
|
|
/// Every call, in call order.
|
|
calls: RefCell<Vec<RecordedCall>>,
|
|
}
|
|
|
|
impl MockClient {
|
|
fn new(responses: Vec<Result<String, LlmError>>) -> Self {
|
|
Self {
|
|
responses: RefCell::new(responses),
|
|
calls: RefCell::new(Vec::new()),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl LlmClient for MockClient {
|
|
fn post_json(
|
|
&self,
|
|
url: &str,
|
|
body: &str,
|
|
headers: &[(&'static str, String)],
|
|
) -> Result<String, LlmError> {
|
|
self.calls.borrow_mut().push((
|
|
url.to_string(),
|
|
body.to_string(),
|
|
headers
|
|
.iter()
|
|
.map(|(k, v)| (k.to_string(), v.clone()))
|
|
.collect(),
|
|
));
|
|
self.responses
|
|
.borrow_mut()
|
|
.pop()
|
|
.unwrap_or(Err(LlmError::Http("no more mock responses".into())))
|
|
}
|
|
}
|
|
|
|
fn params<'a>(api: Api) -> ChatParams<'a> {
|
|
ChatParams {
|
|
api,
|
|
endpoint: "http://x/api/chat",
|
|
model: "qwen",
|
|
api_key: "",
|
|
system_prompt: "",
|
|
prompt: "hello",
|
|
keep_alive: "",
|
|
sampling: Sampling::default(),
|
|
}
|
|
}
|
|
|
|
// ---------------- api input parsing ----------------
|
|
|
|
#[test]
|
|
fn api_parse_defaults_to_ollama() {
|
|
assert_eq!(Api::parse("").unwrap(), Api::Ollama);
|
|
assert_eq!(Api::parse("ollama").unwrap(), Api::Ollama);
|
|
assert_eq!(Api::parse(" OpenAI ").unwrap(), Api::Openai);
|
|
assert_eq!(Api::parse("anthropic").unwrap(), Api::Anthropic);
|
|
}
|
|
|
|
#[test]
|
|
fn api_parse_rejects_unknown() {
|
|
assert!(matches!(
|
|
Api::parse("gemini"),
|
|
Err(LlmError::UnsupportedApi(_))
|
|
));
|
|
}
|
|
|
|
// ---------------- Ollama wire format (regression) ----------------
|
|
|
|
#[test]
|
|
fn ollama_body_includes_system_when_provided() {
|
|
let p = ChatParams {
|
|
system_prompt: "be helpful",
|
|
..params(Api::Ollama)
|
|
};
|
|
let body = build_ollama_body(&p).expect("body builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
let messages = v["messages"].as_array().unwrap();
|
|
assert_eq!(messages.len(), 2);
|
|
assert_eq!(messages[0]["role"], "system");
|
|
assert_eq!(messages[0]["content"], "be helpful");
|
|
assert_eq!(messages[1]["role"], "user");
|
|
assert_eq!(messages[1]["content"], "hello");
|
|
assert_eq!(v["stream"], false);
|
|
}
|
|
|
|
#[test]
|
|
fn ollama_body_omits_system_when_empty() {
|
|
let body = build_ollama_body(¶ms(Api::Ollama)).expect("body builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
let messages = v["messages"].as_array().unwrap();
|
|
assert_eq!(messages.len(), 1);
|
|
assert_eq!(messages[0]["role"], "user");
|
|
}
|
|
|
|
#[test]
|
|
fn extract_content_pulls_message_content() {
|
|
let body = r#"{"message":{"role":"assistant","content":"hi there"},"done":true}"#;
|
|
assert_eq!(extract_ollama_content(body).unwrap(), "hi there");
|
|
}
|
|
|
|
#[test]
|
|
fn extract_content_errors_on_missing_field() {
|
|
let body = r#"{"done":true}"#;
|
|
assert!(matches!(
|
|
extract_ollama_content(body),
|
|
Err(LlmError::Decode(_))
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn ollama_bearer_header_only_with_key() {
|
|
assert!(build_headers(¶ms(Api::Ollama)).is_empty());
|
|
let p = ChatParams {
|
|
api_key: "sk-test",
|
|
..params(Api::Ollama)
|
|
};
|
|
assert_eq!(
|
|
build_headers(&p),
|
|
vec![("Authorization", "Bearer sk-test".to_string())]
|
|
);
|
|
}
|
|
|
|
// ---------------- OpenAI wire format ----------------
|
|
|
|
#[test]
|
|
fn openai_body_matches_chat_completions_format() {
|
|
let p = ChatParams {
|
|
api: Api::Openai,
|
|
endpoint: "http://vllm:8000/v1/chat/completions",
|
|
model: "meta-llama/Llama-3.1-8B-Instruct",
|
|
api_key: "sk-x",
|
|
system_prompt: "be terse",
|
|
prompt: "hello",
|
|
keep_alive: "",
|
|
sampling: Sampling::default(),
|
|
};
|
|
let body = build_openai_body(&p).expect("body builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct");
|
|
assert_eq!(v["stream"], false);
|
|
let messages = v["messages"].as_array().unwrap();
|
|
assert_eq!(messages.len(), 2);
|
|
assert_eq!(messages[0]["role"], "system");
|
|
assert_eq!(messages[0]["content"], "be terse");
|
|
assert_eq!(messages[1]["role"], "user");
|
|
assert_eq!(messages[1]["content"], "hello");
|
|
// No Anthropic-only fields leak into the OpenAI body.
|
|
assert!(v.get("max_tokens").is_none());
|
|
assert!(v.get("system").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn openai_response_parses_fixture() {
|
|
// Shape as returned by OpenAI / vLLM /v1/chat/completions.
|
|
let body = r#"{
|
|
"id": "chatcmpl-123",
|
|
"object": "chat.completion",
|
|
"created": 1719000000,
|
|
"model": "meta-llama/Llama-3.1-8B-Instruct",
|
|
"choices": [{
|
|
"index": 0,
|
|
"message": {"role": "assistant", "content": "Hi from vLLM"},
|
|
"finish_reason": "stop"
|
|
}],
|
|
"usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}
|
|
}"#;
|
|
assert_eq!(extract_openai_content(body).unwrap(), "Hi from vLLM");
|
|
}
|
|
|
|
#[test]
|
|
fn openai_response_errors_on_missing_choices() {
|
|
assert!(matches!(
|
|
extract_openai_content(r#"{"object":"chat.completion","choices":[]}"#),
|
|
Err(LlmError::Decode(_))
|
|
));
|
|
assert!(matches!(
|
|
extract_openai_content(r#"{"error":{"message":"invalid key"}}"#),
|
|
Err(LlmError::Decode(_))
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn openai_uses_bearer_auth() {
|
|
let p = ChatParams {
|
|
api_key: "sk-test",
|
|
..params(Api::Openai)
|
|
};
|
|
assert_eq!(
|
|
build_headers(&p),
|
|
vec![("Authorization", "Bearer sk-test".to_string())]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn openai_chat_returns_content_and_no_digest() {
|
|
let canned = r#"{"choices":[{"message":{"role":"assistant","content":"ok"}}]}"#;
|
|
let client = MockClient::new(vec![Ok(canned.to_string())]);
|
|
let p = ChatParams {
|
|
api: Api::Openai,
|
|
endpoint: "http://vllm:8000/v1/chat/completions",
|
|
model: "m",
|
|
api_key: "sk",
|
|
system_prompt: "",
|
|
prompt: "hi",
|
|
keep_alive: "",
|
|
sampling: Sampling::default(),
|
|
};
|
|
let result = chat_with_identity(&client, &p).unwrap();
|
|
assert_eq!(result.response, "ok");
|
|
// No digest API on OpenAI-compatible endpoints — documented
|
|
// empty, and exactly one HTTP call (no probe).
|
|
assert_eq!(result.model_digest, None);
|
|
assert_eq!(client.calls.borrow().len(), 1);
|
|
}
|
|
|
|
// ---------------- Anthropic wire format ----------------
|
|
|
|
#[test]
|
|
fn anthropic_body_matches_messages_format() {
|
|
let p = ChatParams {
|
|
api: Api::Anthropic,
|
|
endpoint: "https://api.anthropic.com/v1/messages",
|
|
model: "claude-fable-5",
|
|
api_key: "sk-ant",
|
|
system_prompt: "be terse",
|
|
prompt: "hello",
|
|
keep_alive: "",
|
|
sampling: Sampling::default(),
|
|
};
|
|
let body = build_anthropic_body(&p).expect("body builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert_eq!(v["model"], "claude-fable-5");
|
|
// max_tokens is mandatory in the Messages API.
|
|
assert_eq!(v["max_tokens"], 4096);
|
|
// system is a top-level field, never a message.
|
|
assert_eq!(v["system"], "be terse");
|
|
let messages = v["messages"].as_array().unwrap();
|
|
assert_eq!(messages.len(), 1);
|
|
assert_eq!(messages[0]["role"], "user");
|
|
assert_eq!(messages[0]["content"], "hello");
|
|
// No OpenAI/Ollama-only fields leak in.
|
|
assert!(v.get("stream").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_body_omits_system_when_empty() {
|
|
let body = build_anthropic_body(¶ms(Api::Anthropic)).expect("body builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert!(v.get("system").is_none());
|
|
assert_eq!(v["max_tokens"], 4096);
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_response_parses_fixture() {
|
|
// Shape as returned by the Anthropic Messages API.
|
|
let body = r#"{
|
|
"id": "msg_01XFDUDYJgAACzvnptvVoYEL",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"model": "claude-fable-5",
|
|
"content": [{"type": "text", "text": "Hi from Claude"}],
|
|
"stop_reason": "end_turn",
|
|
"usage": {"input_tokens": 10, "output_tokens": 5}
|
|
}"#;
|
|
assert_eq!(extract_anthropic_content(body).unwrap(), "Hi from Claude");
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_response_errors_on_missing_content() {
|
|
assert!(matches!(
|
|
extract_anthropic_content(r#"{"type":"message","content":[]}"#),
|
|
Err(LlmError::Decode(_))
|
|
));
|
|
assert!(matches!(
|
|
extract_anthropic_content(r#"{"type":"error","error":{"message":"x"}}"#),
|
|
Err(LlmError::Decode(_))
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_uses_x_api_key_and_version_headers() {
|
|
let p = ChatParams {
|
|
api_key: "sk-ant-test",
|
|
..params(Api::Anthropic)
|
|
};
|
|
let headers = build_headers(&p);
|
|
assert_eq!(
|
|
headers,
|
|
vec![
|
|
("x-api-key", "sk-ant-test".to_string()),
|
|
("anthropic-version", "2023-06-01".to_string()),
|
|
]
|
|
);
|
|
// Never Bearer auth on the Anthropic path.
|
|
assert!(headers.iter().all(|(k, _)| *k != "Authorization"));
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_chat_returns_content_and_no_digest() {
|
|
let canned = r#"{"content":[{"type":"text","text":"ok"}]}"#;
|
|
let client = MockClient::new(vec![Ok(canned.to_string())]);
|
|
let p = ChatParams {
|
|
api: Api::Anthropic,
|
|
endpoint: "https://api.anthropic.com/v1/messages",
|
|
model: "claude-fable-5",
|
|
api_key: "sk-ant",
|
|
system_prompt: "",
|
|
prompt: "hi",
|
|
keep_alive: "",
|
|
sampling: Sampling::default(),
|
|
};
|
|
let result = chat_with_identity(&client, &p).unwrap();
|
|
assert_eq!(result.response, "ok");
|
|
assert_eq!(result.model_digest, None);
|
|
assert_eq!(client.calls.borrow().len(), 1);
|
|
}
|
|
|
|
// ---------------- digest probe (Ollama only) ----------------
|
|
|
|
#[test]
|
|
fn derive_show_url_swaps_chat_suffix() {
|
|
assert_eq!(
|
|
derive_show_url("http://localhost:11434/api/chat").as_deref(),
|
|
Some("http://localhost:11434/api/show")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn derive_show_url_returns_none_for_non_ollama() {
|
|
assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn extract_show_digest_finds_top_level() {
|
|
let body = r#"{"digest":"sha256:abc"}"#;
|
|
assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:abc"));
|
|
}
|
|
|
|
#[test]
|
|
fn extract_show_digest_falls_back_to_nested() {
|
|
let body = r#"{"details":{"digest":"sha256:nested"}}"#;
|
|
assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:nested"));
|
|
}
|
|
|
|
#[test]
|
|
fn chat_with_identity_records_digest_when_show_responds() {
|
|
let canned_chat = r#"{"message":{"content":"the response"},"done":true}"#.to_string();
|
|
let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string();
|
|
let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]);
|
|
let p = ChatParams {
|
|
endpoint: "http://localhost:11434/api/chat",
|
|
..params(Api::Ollama)
|
|
};
|
|
let result = chat_with_identity(&client, &p).unwrap();
|
|
assert_eq!(result.response, "the response");
|
|
assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef"));
|
|
}
|
|
|
|
#[test]
|
|
fn chat_with_identity_swallows_show_failure() {
|
|
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();
|
|
let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]);
|
|
let p = ChatParams {
|
|
endpoint: "http://localhost:11434/api/chat",
|
|
..params(Api::Ollama)
|
|
};
|
|
let result = chat_with_identity(&client, &p).unwrap();
|
|
assert_eq!(result.response, "ok");
|
|
assert_eq!(result.model_digest, None);
|
|
}
|
|
|
|
#[test]
|
|
fn chat_with_identity_skips_probe_for_non_ollama_url() {
|
|
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();
|
|
let client = MockClient::new(vec![Ok(canned_chat)]);
|
|
let p = ChatParams {
|
|
endpoint: "https://example.com/proxy/chat",
|
|
..params(Api::Ollama)
|
|
};
|
|
let result = chat_with_identity(&client, &p).unwrap();
|
|
assert_eq!(result.response, "ok");
|
|
assert_eq!(result.model_digest, None);
|
|
}
|
|
|
|
// ---------------- input guards ----------------
|
|
|
|
#[test]
|
|
fn empty_prompt_is_rejected() {
|
|
let client = MockClient::new(vec![]);
|
|
let p = ChatParams {
|
|
prompt: "",
|
|
..params(Api::Ollama)
|
|
};
|
|
assert!(matches!(
|
|
chat_with_identity(&client, &p),
|
|
Err(LlmError::MissingInput("prompt"))
|
|
));
|
|
}
|
|
|
|
/// Guard: `api: vllm` exists ONLY for operator clarity — the
|
|
/// request it emits (URL, body, headers) must stay identical to
|
|
/// `api: openai`. If the two ever diverge, that is a new wire
|
|
/// dialect and needs its own deliberate design, not drift.
|
|
#[test]
|
|
fn vllm_is_the_openai_wire_verbatim() {
|
|
assert_eq!(Api::parse("vllm").unwrap(), Api::Vllm);
|
|
let openai_reply = r#"{"choices":[{"message":{"role":"assistant","content":"x"}}]}"#;
|
|
let recorded: Vec<RecordedCall> = [Api::Vllm, Api::Openai]
|
|
.into_iter()
|
|
.map(|api| {
|
|
let client = MockClient::new(vec![Ok(openai_reply.to_string())]);
|
|
chat_with_identity(&client, ¶ms(api)).unwrap();
|
|
client.calls.into_inner().pop().unwrap()
|
|
})
|
|
.collect();
|
|
assert_eq!(recorded[0], recorded[1], "vllm request drifted from openai");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod keep_alive_tests {
|
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
use super::*;
|
|
|
|
fn p(api: Api, keep_alive: &str) -> ChatParams<'_> {
|
|
ChatParams {
|
|
api,
|
|
endpoint: "http://x/api/chat",
|
|
model: "qwen",
|
|
api_key: "",
|
|
system_prompt: "",
|
|
prompt: "hello",
|
|
keep_alive,
|
|
sampling: Sampling::default(),
|
|
}
|
|
}
|
|
|
|
/// The reason this input exists: Ollama unloads a model after
|
|
/// five minutes, so a demonstrator left idle pays a 30 to 40
|
|
/// second reload on the next first question.
|
|
#[test]
|
|
fn a_duration_reaches_the_ollama_body() {
|
|
let body = build_ollama_body(&p(Api::Ollama, "4h")).expect("body builds");
|
|
assert!(
|
|
body.contains(r#""keep_alive":"4h""#),
|
|
"keep_alive missing from the ollama body: {body}"
|
|
);
|
|
}
|
|
|
|
/// Ollama reads -1 as "never unload" only as a NUMBER. Quoted,
|
|
/// it is parsed as a duration string and the setting silently
|
|
/// does nothing, which is the failure this input exists to
|
|
/// prevent.
|
|
#[test]
|
|
fn minus_one_is_sent_as_a_number_not_a_string() {
|
|
let body = build_ollama_body(&p(Api::Ollama, "-1")).expect("body builds");
|
|
assert!(
|
|
body.contains(r#""keep_alive":-1"#),
|
|
"expected a numeric -1, got: {body}"
|
|
);
|
|
assert!(
|
|
!body.contains(r#""keep_alive":"-1""#),
|
|
"-1 was quoted, which Ollama reads as a duration: {body}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn plain_seconds_are_sent_as_a_number() {
|
|
let body = build_ollama_body(&p(Api::Ollama, "600")).expect("body builds");
|
|
assert!(body.contains(r#""keep_alive":600"#), "{body}");
|
|
}
|
|
|
|
/// Unset must leave the request byte-identical to the previous
|
|
/// version, so nobody's existing flow changes behaviour.
|
|
#[test]
|
|
fn unset_omits_the_field_entirely() {
|
|
let body = build_ollama_body(&p(Api::Ollama, "")).expect("body builds");
|
|
assert!(!body.contains("keep_alive"), "{body}");
|
|
assert_eq!(
|
|
body,
|
|
build_ollama_body(&p(Api::Ollama, " ")).expect("body builds")
|
|
);
|
|
}
|
|
|
|
/// The other wire formats have no such field. lib.rs refuses the
|
|
/// input before reaching here; this pins that the body builders
|
|
/// would not smuggle it through either.
|
|
#[test]
|
|
fn other_wire_formats_carry_no_keep_alive() {
|
|
for api in [Api::Openai, Api::Vllm, Api::Anthropic] {
|
|
let body = match api {
|
|
Api::Anthropic => build_anthropic_body(&p(api, "4h")).expect("body builds"),
|
|
_ => build_openai_body(&p(api, "4h")).expect("body builds"),
|
|
};
|
|
assert!(
|
|
!body.contains("keep_alive"),
|
|
"{api:?} body carries keep_alive: {body}"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod sampling_tests {
|
|
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
|
use super::*;
|
|
|
|
fn p(api: Api, sampling: Sampling<'_>) -> ChatParams<'_> {
|
|
ChatParams {
|
|
api,
|
|
endpoint: "http://x",
|
|
model: "qwen",
|
|
api_key: "",
|
|
system_prompt: "",
|
|
prompt: "hello",
|
|
keep_alive: "",
|
|
sampling,
|
|
}
|
|
}
|
|
|
|
/// What aila needs: a repeatable run. Until now they baked the
|
|
/// values into a derived Ollama model to get one at all.
|
|
#[test]
|
|
fn ollama_nests_sampling_under_options() {
|
|
let s = Sampling {
|
|
temperature: "0",
|
|
seed: "42",
|
|
top_k: "40",
|
|
..Sampling::default()
|
|
};
|
|
let body = build_ollama_body(&p(Api::Ollama, s)).expect("builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert_eq!(v["options"]["temperature"], 0.0);
|
|
assert_eq!(v["options"]["seed"], 42);
|
|
assert_eq!(v["options"]["top_k"], 40);
|
|
}
|
|
|
|
/// Unset must leave the request exactly as it was, so no
|
|
/// existing caller sees a changed body.
|
|
#[test]
|
|
fn nothing_set_means_no_options_block() {
|
|
let body = build_ollama_body(&p(Api::Ollama, Sampling::default())).expect("builds");
|
|
assert!(!body.contains("options"), "{body}");
|
|
}
|
|
|
|
#[test]
|
|
fn openai_puts_sampling_at_the_top_level() {
|
|
let s = Sampling {
|
|
temperature: "0.2",
|
|
top_p: "0.9",
|
|
seed: "7",
|
|
..Sampling::default()
|
|
};
|
|
let body = build_openai_body(&p(Api::Openai, s)).expect("builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert_eq!(v["temperature"], 0.2);
|
|
assert_eq!(v["top_p"], 0.9);
|
|
assert_eq!(v["seed"], 7);
|
|
}
|
|
|
|
/// An option the wire does not have is refused, not dropped: a
|
|
/// caller who set it expects it to apply, and silently ignoring
|
|
/// it produces a run that looks configured and is not.
|
|
#[test]
|
|
fn openai_refuses_top_k() {
|
|
let s = Sampling {
|
|
top_k: "40",
|
|
..Sampling::default()
|
|
};
|
|
let err = build_openai_body(&p(Api::Openai, s)).unwrap_err();
|
|
assert!(err.to_string().contains("top_k"), "{err}");
|
|
}
|
|
|
|
/// Anthropic has no seed, and seed is exactly what someone
|
|
/// chasing a repeatable run is after.
|
|
#[test]
|
|
fn anthropic_refuses_seed() {
|
|
let s = Sampling {
|
|
seed: "42",
|
|
..Sampling::default()
|
|
};
|
|
let err = build_anthropic_body(&p(Api::Anthropic, s)).unwrap_err();
|
|
assert!(err.to_string().contains("seed"), "{err}");
|
|
}
|
|
|
|
#[test]
|
|
fn anthropic_takes_temperature_and_top_k() {
|
|
let s = Sampling {
|
|
temperature: "0.5",
|
|
top_k: "10",
|
|
..Sampling::default()
|
|
};
|
|
let body = build_anthropic_body(&p(Api::Anthropic, s)).expect("builds");
|
|
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
|
|
assert_eq!(v["temperature"], 0.5);
|
|
assert_eq!(v["top_k"], 10);
|
|
}
|
|
|
|
/// A typo must not pass as "server default". It would look like
|
|
/// the setting applied.
|
|
#[test]
|
|
fn a_non_numeric_value_is_an_error() {
|
|
let s = Sampling {
|
|
temperature: "heiss",
|
|
..Sampling::default()
|
|
};
|
|
let err = build_ollama_body(&p(Api::Ollama, s)).unwrap_err();
|
|
assert!(err.to_string().contains("temperature"), "{err}");
|
|
}
|
|
|
|
/// `seed` must stay a whole number: 42.0 is not a seed.
|
|
#[test]
|
|
fn seed_must_be_a_whole_number() {
|
|
let s = Sampling {
|
|
seed: "42.5",
|
|
..Sampling::default()
|
|
};
|
|
assert!(build_ollama_body(&p(Api::Ollama, s)).is_err());
|
|
}
|
|
}
|