//! Multi-API chat client: Ollama (default), OpenAI, Anthropic. //! //! The wire format is selected via the optional `api` input: //! //! - `ollama` (default) — Ollama `/api/chat`; unchanged v0.1.0 //! behavior including the best-effort model-digest probe. //! - `openai` — OpenAI Chat Completions wire format //! (`/v1/chat/completions`), the de-facto standard implemented //! by vLLM and most self-hosted inference servers. Bearer auth. //! - `anthropic` — Anthropic Messages API (`/v1/messages`), //! `x-api-key` + `anthropic-version` headers, top-level //! `system` field, mandatory `max_tokens`. //! //! No streaming, no tool calls — one prompt in, one completion out. //! //! All HTTP I/O lives behind a `LlmClient` trait so unit tests can //! exercise request building, response parsing, and the digest //! probe on the host without making real network calls. use serde::Serialize; /// Anthropic's Messages API requires `max_tokens`. This default is /// large enough for document-processing completions while staying /// well below every current model's output cap. const ANTHROPIC_DEFAULT_MAX_TOKENS: u32 = 4096; /// Pinned Messages API version header. Anthropic keeps old /// versions working; bump deliberately, never implicitly. const ANTHROPIC_VERSION: &str = "2023-06-01"; #[allow(dead_code)] #[derive(Debug, thiserror::Error)] pub enum LlmError { #[error("missing required input '{0}'")] MissingInput(&'static str), #[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")] UnsupportedApi(String), #[error("http error: {0}")] Http(String), #[error("non-success status: {0}")] Status(u16), #[error("response body could not be parsed as the expected schema: {0}")] Decode(String), } /// Which wire format to speak. Selected by the `api` input. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Api { Ollama, Openai, Anthropic, } impl Api { /// Parse the `api` input. Empty string means the input was not /// provided and falls back to Ollama (v0.1.0 behavior). pub fn parse(raw: &str) -> Result { match raw.trim().to_ascii_lowercase().as_str() { "" | "ollama" => Ok(Api::Ollama), "openai" => Ok(Api::Openai), "anthropic" => Ok(Api::Anthropic), other => Err(LlmError::UnsupportedApi(other.to_string())), } } } pub trait LlmClient { /// POST `body` as JSON to `url` with the given extra headers. /// `Content-Type: application/json` is implied. Header VALUES /// may carry credentials — implementations must never log them. fn post_json( &self, url: &str, body: &str, headers: &[(&'static str, String)], ) -> Result; } #[derive(Debug, Clone)] pub struct ChatParams<'a> { pub api: Api, pub endpoint: &'a str, pub model: &'a str, pub api_key: &'a str, pub system_prompt: &'a str, pub prompt: &'a str, } /// Auth/protocol headers for the selected API. The api_key is only /// ever placed into a header value here; it must not appear in any /// error, output, or log line. pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> { let mut headers = Vec::with_capacity(2); match p.api { Api::Ollama | Api::Openai => { if !p.api_key.is_empty() { headers.push(("Authorization", format!("Bearer {}", p.api_key))); } } Api::Anthropic => { if !p.api_key.is_empty() { headers.push(("x-api-key", p.api_key.to_string())); } headers.push(("anthropic-version", ANTHROPIC_VERSION.to_string())); } } headers } #[derive(Serialize)] struct ChatMessage<'a> { role: &'a str, content: &'a str, } #[derive(Serialize)] struct OllamaRequest<'a> { model: &'a str, messages: Vec>, stream: bool, } #[derive(Serialize)] struct OpenAiRequest<'a> { model: &'a str, messages: Vec>, stream: bool, } #[derive(Serialize)] struct AnthropicRequest<'a> { model: &'a str, max_tokens: u32, #[serde(skip_serializing_if = "Option::is_none")] system: Option<&'a str>, messages: Vec>, } /// System-then-user message list shared by the Ollama and OpenAI /// wire formats. The system message is omitted when empty so a /// deployment can use the model's built-in system prompt. fn build_messages<'a>(p: &ChatParams<'a>) -> Vec> { let mut messages = Vec::with_capacity(2); if !p.system_prompt.is_empty() { messages.push(ChatMessage { role: "system", content: p.system_prompt, }); } messages.push(ChatMessage { role: "user", content: p.prompt, }); messages } /// Build the request body for an Ollama `/api/chat` invocation. pub fn build_ollama_body(p: &ChatParams) -> String { let req = OllamaRequest { model: p.model, messages: build_messages(p), stream: false, }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } /// Build the request body for an OpenAI-compatible /// `/v1/chat/completions` invocation (OpenAI, vLLM, most /// self-hosted inference servers). pub fn build_openai_body(p: &ChatParams) -> String { let req = OpenAiRequest { model: p.model, messages: build_messages(p), stream: false, }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } /// Build the request body for an Anthropic `/v1/messages` /// invocation. `system` is a top-level field (not a message); /// `max_tokens` is mandatory in the Messages API. pub fn build_anthropic_body(p: &ChatParams) -> String { let req = AnthropicRequest { model: p.model, max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, system: if p.system_prompt.is_empty() { None } else { Some(p.system_prompt) }, messages: vec![ChatMessage { role: "user", content: p.prompt, }], }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } /// Extract the assistant's message text from an Ollama /// /api/chat response. pub fn extract_ollama_content(body: &str) -> Result { let v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; v.get("message") .and_then(|m| m.get("content")) .and_then(|c| c.as_str()) .map(|s| s.to_string()) .ok_or_else(|| LlmError::Decode("missing message.content".into())) } /// Extract the assistant text from an OpenAI Chat Completions /// response: `choices[0].message.content`. pub fn extract_openai_content(body: &str) -> Result { let v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; v.get("choices") .and_then(|c| c.get(0)) .and_then(|c| c.get("message")) .and_then(|m| m.get("content")) .and_then(|c| c.as_str()) .map(|s| s.to_string()) .ok_or_else(|| LlmError::Decode("missing choices[0].message.content".into())) } /// Extract the assistant text from an Anthropic Messages /// response: `content[0].text`. pub fn extract_anthropic_content(body: &str) -> Result { let v: serde_json::Value = serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; v.get("content") .and_then(|c| c.get(0)) .and_then(|c| c.get("text")) .and_then(|t| t.as_str()) .map(|s| s.to_string()) .ok_or_else(|| LlmError::Decode("missing content[0].text".into())) } #[derive(Debug, Clone)] pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, } /// Run one chat call in the selected wire format AND, for Ollama, /// probe the model digest. The probe is best-effort — non-Ollama /// APIs expose no digest endpoint, so `model_digest = None` there /// (documented empty, never invented); transient probe failures on /// Ollama also yield `None` rather than failing the whole call. pub fn chat_with_identity( client: &C, p: &ChatParams, ) -> Result { if p.endpoint.is_empty() { return Err(LlmError::MissingInput("endpoint")); } if p.model.is_empty() { return Err(LlmError::MissingInput("model")); } if p.prompt.is_empty() { return Err(LlmError::MissingInput("prompt")); } let body = match p.api { Api::Ollama => build_ollama_body(p), Api::Openai => build_openai_body(p), Api::Anthropic => build_anthropic_body(p), }; let headers = build_headers(p); let response_body = client.post_json(p.endpoint, &body, &headers)?; let response = match p.api { Api::Ollama => extract_ollama_content(&response_body)?, Api::Openai => extract_openai_content(&response_body)?, Api::Anthropic => extract_anthropic_content(&response_body)?, }; let model_digest = match p.api { Api::Ollama => probe_model_digest(client, p), Api::Openai | Api::Anthropic => None, }; Ok(ChatWithIdentity { response, model_digest, }) } fn probe_model_digest(client: &C, p: &ChatParams) -> Option { let show_url = derive_show_url(p.endpoint)?; let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?; let headers = build_headers(p); let response = client.post_json(&show_url, &body, &headers).ok()?; extract_show_digest(&response) } pub fn derive_show_url(chat_endpoint: &str) -> Option { if chat_endpoint.ends_with("/api/chat") { let head_len = chat_endpoint.len() - "/api/chat".len(); let mut url = String::with_capacity(head_len + "/api/show".len()); url.push_str(&chat_endpoint[..head_len]); url.push_str("/api/show"); Some(url) } else { None } } pub fn extract_show_digest(body: &str) -> Option { let v: serde_json::Value = serde_json::from_str(body).ok()?; let candidates = [ v.get("digest"), v.get("details").and_then(|d| d.get("digest")), v.get("model_info").and_then(|d| d.get("digest")), ]; for cand in candidates { if let Some(s) = cand.and_then(|v| v.as_str()) { if !s.is_empty() { return Some(s.to_string()); } } } None } #[cfg(test)] mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; use std::cell::RefCell; /// (url, body, headers) of one recorded call. type RecordedCall = (String, String, Vec<(String, String)>); struct MockClient { responses: RefCell>>, /// Every call, in call order. calls: RefCell>, } impl MockClient { fn new(responses: Vec>) -> Self { Self { responses: RefCell::new(responses), calls: RefCell::new(Vec::new()), } } } impl LlmClient for MockClient { fn post_json( &self, url: &str, body: &str, headers: &[(&'static str, String)], ) -> Result { self.calls.borrow_mut().push(( url.to_string(), body.to_string(), headers .iter() .map(|(k, v)| (k.to_string(), v.clone())) .collect(), )); self.responses .borrow_mut() .pop() .unwrap_or(Err(LlmError::Http("no more mock responses".into()))) } } fn params<'a>(api: Api) -> ChatParams<'a> { ChatParams { api, endpoint: "http://x/api/chat", model: "qwen", api_key: "", system_prompt: "", prompt: "hello", } } // ---------------- api input parsing ---------------- #[test] fn api_parse_defaults_to_ollama() { assert_eq!(Api::parse("").unwrap(), Api::Ollama); assert_eq!(Api::parse("ollama").unwrap(), Api::Ollama); assert_eq!(Api::parse(" OpenAI ").unwrap(), Api::Openai); assert_eq!(Api::parse("anthropic").unwrap(), Api::Anthropic); } #[test] fn api_parse_rejects_unknown() { assert!(matches!( Api::parse("gemini"), Err(LlmError::UnsupportedApi(_)) )); } // ---------------- Ollama wire format (regression) ---------------- #[test] fn ollama_body_includes_system_when_provided() { let p = ChatParams { system_prompt: "be helpful", ..params(Api::Ollama) }; let body = build_ollama_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 2); assert_eq!(messages[0]["role"], "system"); assert_eq!(messages[0]["content"], "be helpful"); assert_eq!(messages[1]["role"], "user"); assert_eq!(messages[1]["content"], "hello"); assert_eq!(v["stream"], false); } #[test] fn ollama_body_omits_system_when_empty() { let body = build_ollama_body(¶ms(Api::Ollama)); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 1); assert_eq!(messages[0]["role"], "user"); } #[test] fn extract_content_pulls_message_content() { let body = r#"{"message":{"role":"assistant","content":"hi there"},"done":true}"#; assert_eq!(extract_ollama_content(body).unwrap(), "hi there"); } #[test] fn extract_content_errors_on_missing_field() { let body = r#"{"done":true}"#; assert!(matches!( extract_ollama_content(body), Err(LlmError::Decode(_)) )); } #[test] fn ollama_bearer_header_only_with_key() { assert!(build_headers(¶ms(Api::Ollama)).is_empty()); let p = ChatParams { api_key: "sk-test", ..params(Api::Ollama) }; assert_eq!( build_headers(&p), vec![("Authorization", "Bearer sk-test".to_string())] ); } // ---------------- OpenAI wire format ---------------- #[test] fn openai_body_matches_chat_completions_format() { let p = ChatParams { api: Api::Openai, endpoint: "http://vllm:8000/v1/chat/completions", model: "meta-llama/Llama-3.1-8B-Instruct", api_key: "sk-x", system_prompt: "be terse", prompt: "hello", }; let body = build_openai_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct"); assert_eq!(v["stream"], false); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 2); assert_eq!(messages[0]["role"], "system"); assert_eq!(messages[0]["content"], "be terse"); assert_eq!(messages[1]["role"], "user"); assert_eq!(messages[1]["content"], "hello"); // No Anthropic-only fields leak into the OpenAI body. assert!(v.get("max_tokens").is_none()); assert!(v.get("system").is_none()); } #[test] fn openai_response_parses_fixture() { // Shape as returned by OpenAI / vLLM /v1/chat/completions. let body = r#"{ "id": "chatcmpl-123", "object": "chat.completion", "created": 1719000000, "model": "meta-llama/Llama-3.1-8B-Instruct", "choices": [{ "index": 0, "message": {"role": "assistant", "content": "Hi from vLLM"}, "finish_reason": "stop" }], "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13} }"#; assert_eq!(extract_openai_content(body).unwrap(), "Hi from vLLM"); } #[test] fn openai_response_errors_on_missing_choices() { assert!(matches!( extract_openai_content(r#"{"object":"chat.completion","choices":[]}"#), Err(LlmError::Decode(_)) )); assert!(matches!( extract_openai_content(r#"{"error":{"message":"invalid key"}}"#), Err(LlmError::Decode(_)) )); } #[test] fn openai_uses_bearer_auth() { let p = ChatParams { api_key: "sk-test", ..params(Api::Openai) }; assert_eq!( build_headers(&p), vec![("Authorization", "Bearer sk-test".to_string())] ); } #[test] fn openai_chat_returns_content_and_no_digest() { let canned = r#"{"choices":[{"message":{"role":"assistant","content":"ok"}}]}"#; let client = MockClient::new(vec![Ok(canned.to_string())]); let p = ChatParams { api: Api::Openai, endpoint: "http://vllm:8000/v1/chat/completions", model: "m", api_key: "sk", system_prompt: "", prompt: "hi", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); // No digest API on OpenAI-compatible endpoints — documented // empty, and exactly one HTTP call (no probe). assert_eq!(result.model_digest, None); assert_eq!(client.calls.borrow().len(), 1); } // ---------------- Anthropic wire format ---------------- #[test] fn anthropic_body_matches_messages_format() { let p = ChatParams { api: Api::Anthropic, endpoint: "https://api.anthropic.com/v1/messages", model: "claude-fable-5", api_key: "sk-ant", system_prompt: "be terse", prompt: "hello", }; let body = build_anthropic_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert_eq!(v["model"], "claude-fable-5"); // max_tokens is mandatory in the Messages API. assert_eq!(v["max_tokens"], 4096); // system is a top-level field, never a message. assert_eq!(v["system"], "be terse"); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 1); assert_eq!(messages[0]["role"], "user"); assert_eq!(messages[0]["content"], "hello"); // No OpenAI/Ollama-only fields leak in. assert!(v.get("stream").is_none()); } #[test] fn anthropic_body_omits_system_when_empty() { let body = build_anthropic_body(¶ms(Api::Anthropic)); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); assert!(v.get("system").is_none()); assert_eq!(v["max_tokens"], 4096); } #[test] fn anthropic_response_parses_fixture() { // Shape as returned by the Anthropic Messages API. let body = r#"{ "id": "msg_01XFDUDYJgAACzvnptvVoYEL", "type": "message", "role": "assistant", "model": "claude-fable-5", "content": [{"type": "text", "text": "Hi from Claude"}], "stop_reason": "end_turn", "usage": {"input_tokens": 10, "output_tokens": 5} }"#; assert_eq!(extract_anthropic_content(body).unwrap(), "Hi from Claude"); } #[test] fn anthropic_response_errors_on_missing_content() { assert!(matches!( extract_anthropic_content(r#"{"type":"message","content":[]}"#), Err(LlmError::Decode(_)) )); assert!(matches!( extract_anthropic_content(r#"{"type":"error","error":{"message":"x"}}"#), Err(LlmError::Decode(_)) )); } #[test] fn anthropic_uses_x_api_key_and_version_headers() { let p = ChatParams { api_key: "sk-ant-test", ..params(Api::Anthropic) }; let headers = build_headers(&p); assert_eq!( headers, vec![ ("x-api-key", "sk-ant-test".to_string()), ("anthropic-version", "2023-06-01".to_string()), ] ); // Never Bearer auth on the Anthropic path. assert!(headers.iter().all(|(k, _)| *k != "Authorization")); } #[test] fn anthropic_chat_returns_content_and_no_digest() { let canned = r#"{"content":[{"type":"text","text":"ok"}]}"#; let client = MockClient::new(vec![Ok(canned.to_string())]); let p = ChatParams { api: Api::Anthropic, endpoint: "https://api.anthropic.com/v1/messages", model: "claude-fable-5", api_key: "sk-ant", system_prompt: "", prompt: "hi", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); assert_eq!(result.model_digest, None); assert_eq!(client.calls.borrow().len(), 1); } // ---------------- digest probe (Ollama only) ---------------- #[test] fn derive_show_url_swaps_chat_suffix() { assert_eq!( derive_show_url("http://localhost:11434/api/chat").as_deref(), Some("http://localhost:11434/api/show") ); } #[test] fn derive_show_url_returns_none_for_non_ollama() { assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); } #[test] fn extract_show_digest_finds_top_level() { let body = r#"{"digest":"sha256:abc"}"#; assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:abc")); } #[test] fn extract_show_digest_falls_back_to_nested() { let body = r#"{"details":{"digest":"sha256:nested"}}"#; assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:nested")); } #[test] fn chat_with_identity_records_digest_when_show_responds() { let canned_chat = r#"{"message":{"content":"the response"},"done":true}"#.to_string(); let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "the response"); assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); } #[test] fn chat_with_identity_swallows_show_failure() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); assert_eq!(result.model_digest, None); } #[test] fn chat_with_identity_skips_probe_for_non_ollama_url() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); let client = MockClient::new(vec![Ok(canned_chat)]); let p = ChatParams { endpoint: "https://example.com/proxy/chat", ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); assert_eq!(result.model_digest, None); } // ---------------- input guards ---------------- #[test] fn empty_prompt_is_rejected() { let client = MockClient::new(vec![]); let p = ChatParams { prompt: "", ..params(Api::Ollama) }; assert!(matches!( chat_with_identity(&client, &p), Err(LlmError::MissingInput("prompt")) )); } }