diff --git a/Cargo.lock b/Cargo.lock index dcff588..462390b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,8 +40,8 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chain-module-sdk" -version = "0.1.2" -source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +version = "0.3.0" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?tag=v0.3.0#a4415ada7e76aac5e3aabf3e4c4311ffc348d8d0" dependencies = [ "chain-module-sdk-macros", "serde", @@ -52,8 +52,8 @@ dependencies = [ [[package]] name = "chain-module-sdk-macros" -version = "0.1.2" -source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" +version = "0.3.0" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?tag=v0.3.0#a4415ada7e76aac5e3aabf3e4c4311ffc348d8d0" dependencies = [ "proc-macro2", "quote", @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.1.0" +version = "0.2.0" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index 72db65b..6a761ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.1.0" +version = "0.2.0" edition = "2024" license = "Apache-2.0" publish = false @@ -17,7 +17,7 @@ description = "F∆I module providing llm.chat" crate-type = ["cdylib", "rlib"] [dependencies] -chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", branch = "main" } +chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", tag = "v0.3.0" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/MODULE.de.md b/MODULE.de.md index 4d12be6..ab71365 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -1,23 +1,35 @@ # llm.chat -Generischer Ollama-kompatibler LLM-Chat-Adapter. Das -Baustein-Modul für jeden Flow, der eine LLM-Einzelantwort -braucht — klassifizieren, Felder extrahieren, umformulieren, -entscheiden — ohne einen eigenen HTTP-Client zu bauen. +Generischer LLM-Chat-Adapter. Das Baustein-Modul für jeden +Flow, der eine LLM-Einzelantwort braucht — klassifizieren, +Felder extrahieren, umformulieren, entscheiden — ohne einen +eigenen HTTP-Client zu bauen. Spricht drei Wire-Formate, +wählbar über die optionale Eingabe `api`: Ollama (Default), +OpenAI Chat Completions (OpenAI, vLLM, kompatible Server) und +die Anthropic Messages API. ## Capability -- `llm.chat@0.1.0` +- `llm.chat@0.2.0` ## Eingaben | Name | Typ | Beschreibung | | ---------------- | ---- | ------------------------------------------------------------------------- | | `prompt` | text | Der User-Prompt. | -| `endpoint` | text | Ollama-förmiger `/api/chat`-URL (z.B. `http://localhost:11434/api/chat`). | -| `model` | text | Modell-ID (z.B. `qwen2.5:14b`, `llama3.1:8b`). | -| `api_key` | text | Optionaler Bearer-Token für Cloud-Endpunkte. | +| `endpoint` | text | Chat-Endpunkt-URL passend zur gewählten `api` (siehe unten). | +| `model` | text | Modell-ID (z. B. `qwen2.5:14b`, `claude-fable-5`). | +| `api_key` | text | Optionaler API-Key. Erscheint nie in Ausgaben, Logs oder Events. | | `system_prompt` | text | Optionale System-Nachricht. Leer = Default des Modells. | +| `api` | text | Optionales Wire-Format: `ollama` (Default), `openai`, `anthropic`. | + +### Wire-Formate (`api`) + +| `api` | Endpunkt-Form | Auth-Header | Hinweise | +| ----------- | ----------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | +| `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | Verhalten wie v0.1.0, unverändert. Modell-Digest-Probe via `/api/show`. | +| `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI-Chat-Completions-Format — sprechen auch vLLM und die meisten selbst gehosteten Inferenz-Server. | +| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` ist Top-Level-Feld; `max_tokens` ist Pflicht und steht fest auf 4096. | ## Ausgaben @@ -26,7 +38,7 @@ entscheiden — ohne einen eigenen HTTP-Client zu bauen. | `response` | text | Die Antwort des Assistants als Plain-Text. | | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | -| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei Cloud-APIs. | +| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | Zusammen beantworten die drei `model_*`-Ausgaben die Audit- Frage: „Welches genaue Modell hat diese Antwort erzeugt?" — @@ -43,8 +55,8 @@ permissions: - "net: api.anthropic.com" ``` -Loopback (lokales Ollama) per Default. Cloud-Endpunkte brauchen -einen Operator-Policy-Override in +Loopback (lokales Ollama / vLLM) per Default. Cloud-Endpunkte +brauchen einen Operator-Policy-Override in `~/.chain/config.yaml#security.max_permissions`. ## Warum dieses Modul statt Inline-HTTP @@ -61,23 +73,24 @@ Drei Gründe: `net:`-Liste. Ein hausgemachtes HTTP-Modul würde seine Endpunkte entweder verstecken oder bei jeder Installation eine frische Review-Oberfläche schaffen. -3. **Endpunkt-Portabilität.** Endpunkt + Modell sind Flow- - Inputs, keine Compile-Time-Konstanten. Derselbe Flow läuft - in der Entwicklung gegen `localhost:11434` und in Produktion - gegen ein Inferenz-Cluster — nur die Eingabe wechselt. +3. **Endpunkt-Portabilität.** Endpunkt, Modell und Wire-Format + sind Flow-Inputs, keine Compile-Time-Konstanten. Derselbe + Flow läuft in der Entwicklung gegen `localhost:11434` und + in Produktion gegen ein vLLM-Cluster — nur die Eingaben + wechseln. -## Grenzen in v0.1.0 +## Grenzen in v0.2.0 - Kein Streaming. Die ganze Antwort wird gepuffert, bevor der Output-Step feuert. - Keine Tool-Call- / Function-Call-Oberfläche. Ein Flow, der Tool-Use braucht, kombiniert mehrere `llm.chat`-Schritte mit Prompt-Engineering oder nutzt MCP via Bridge. -- Cloud-Provider-Adapter für OpenAI und Anthropic kommen erst, - wenn ein Flow sie braucht. Heute ist das Ollama-Wire-Format - das einzige Ziel. +- Anthropic-`max_tokens` steht fest auf 4096 (die API verlangt + das Feld; eine konfigurierbare Eingabe kommt, sobald ein + Flow sie braucht). -## Beispiel-Flow +## Beispiel-Flows ```yaml name: classify-incoming @@ -98,6 +111,33 @@ outputs: audit_model: $classify.model_digest ``` +Gegen einen vLLM-Server (OpenAI-kompatibel): + +```yaml +steps: + - id: classify + use: llm.chat@^0 + with: + api: "openai" + endpoint: "http://inference.internal:8000/v1/chat/completions" + model: "meta-llama/Llama-3.1-8B-Instruct" + prompt: $inputs.text +``` + +Gegen die Anthropic Messages API: + +```yaml +steps: + - id: classify + use: llm.chat@^0 + with: + api: "anthropic" + endpoint: "https://api.anthropic.com/v1/messages" + model: "claude-fable-5" + api_key: $inputs.anthropic_key + prompt: $inputs.text +``` + ## Build ```bash diff --git a/MODULE.md b/MODULE.md index 49460ad..cb8c167 100644 --- a/MODULE.md +++ b/MODULE.md @@ -1,23 +1,35 @@ # llm.chat -Generic Ollama-compatible LLM chat adapter. The building-block -module any flow uses when it needs a one-shot LLM completion — -classify, extract-fields, rewrite, decide — instead of rolling -its own HTTP client. +Generic LLM chat adapter. The building-block module any flow +uses when it needs a one-shot LLM completion — classify, +extract-fields, rewrite, decide — instead of rolling its own +HTTP client. Speaks three wire formats, selected by the +optional `api` input: Ollama (default), OpenAI Chat +Completions (OpenAI, vLLM, compatible servers), and the +Anthropic Messages API. ## Capability -- `llm.chat@0.1.0` +- `llm.chat@0.2.0` ## Inputs | Name | Type | Description | | ---------------- | ---- | --------------------------------------------------------------- | | `prompt` | text | The user-facing prompt. | -| `endpoint` | text | Ollama-shaped `/api/chat` URL (e.g. `http://localhost:11434/api/chat`). | -| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `llama3.1:8b`). | -| `api_key` | text | Optional bearer token for cloud-hosted endpoints. | +| `endpoint` | text | Chat endpoint URL matching the selected `api` (see below). | +| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | +| `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | +| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | + +### Wire formats (`api`) + +| `api` | Endpoint shape | Auth header | Notes | +| ----------- | --------------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | +| `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | v0.1.0 behavior, unchanged. Model-digest probe via `/api/show`. | +| `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI Chat Completions format — also spoken by vLLM and most self-hosted inference servers. | +| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` is a top-level field; `max_tokens` is mandatory and defaults to 4096. | ## Outputs @@ -26,7 +38,7 @@ its own HTTP client. | `response` | text | The assistant's plain-text reply. | | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | -| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for cloud APIs. | +| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | Together the three `model_*` outputs answer the audit question: "which exact model produced this response?" — that @@ -43,7 +55,7 @@ permissions: - "net: api.anthropic.com" ``` -Loopback (local Ollama) by default. Cloud endpoints require an +Loopback (local Ollama / vLLM) by default. Cloud endpoints require an operator-policy override in `~/.chain/config.yaml#security.max_permissions`. ## Why this module instead of inline HTTP @@ -59,23 +71,22 @@ Three reasons: the installed modules + its declared `net:` list. A home-grown HTTP module would either hide its endpoints or be a fresh review surface every time. -3. **Endpoint portability.** The endpoint + model are flow - inputs, not compile-time constants. The same flow runs - against `localhost:11434` in dev and a production - inference cluster in prod just by swapping the input. +3. **Endpoint portability.** The endpoint, model, and wire + format are flow inputs, not compile-time constants. The + same flow runs against `localhost:11434` in dev and a + production vLLM cluster in prod just by swapping inputs. -## Limits in v0.1.0 +## Limits in v0.2.0 - No streaming. The whole reply is buffered before the output step fires. - No tool-call / function-call surface. A flow needing tool use composes multiple `llm.chat` steps with prompt engineering, or uses MCP via the bridge. -- Cloud-provider adapters for OpenAI and Anthropic are - deferred until a flow actually needs them. Today the - Ollama wire-format is the only target. +- Anthropic `max_tokens` is fixed at 4096 (the API requires + the field; a configurable input lands when a flow needs it). -## Example flow +## Example flows ```yaml name: classify-incoming @@ -96,6 +107,33 @@ outputs: audit_model: $classify.model_digest ``` +Against a vLLM (OpenAI-compatible) server: + +```yaml +steps: + - id: classify + use: llm.chat@^0 + with: + api: "openai" + endpoint: "http://inference.internal:8000/v1/chat/completions" + model: "meta-llama/Llama-3.1-8B-Instruct" + prompt: $inputs.text +``` + +Against the Anthropic Messages API: + +```yaml +steps: + - id: classify + use: llm.chat@^0 + with: + api: "anthropic" + endpoint: "https://api.anthropic.com/v1/messages" + model: "claude-fable-5" + api_key: $inputs.anthropic_key + prompt: $inputs.text +``` + ## Build ```bash diff --git a/README.md b/README.md index 275de8d..5e55757 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ # llm-chat F∆I module providing the `llm.chat` capability — a generic -Ollama-compatible LLM chat adapter. +LLM chat adapter speaking Ollama (default), OpenAI Chat +Completions (OpenAI, vLLM, compatible servers), and the +Anthropic Messages API, selected via the optional `api` input. ## Capability | Field | Value | |-------|-------| -| Capability | `llm.chat@0.1.0` | -| Inputs | `prompt: text`, `endpoint: text`, `model: text`, `api_key: text` (opt), `system_prompt: text` (opt) | +| Capability | `llm.chat@0.2.0` | +| Inputs | `prompt: text`, `endpoint: text`, `model: text`, `api_key: text` (opt), `system_prompt: text` (opt), `api: text` (opt: `ollama`\|`openai`\|`anthropic`) | | Outputs | `response: text`, `model_endpoint: text`, `model_name: text`, `model_digest: text` | | Permissions | `net: localhost`, `net: 127.0.0.1`, `net: api.openai.com`, `net: api.anthropic.com` | | Status (in store index) | `alpha` | @@ -41,7 +43,7 @@ cargo build --release --target wasm32-wasip2 ## Test ```bash -cargo test # 11 tests, all host-side +cargo test # wire-format unit tests, all host-side cargo build --release --target wasm32-wasip2 ``` diff --git a/module.wasm b/module.wasm index a531ccd..353c547 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 503111a..4472d9d 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.1.0 +version: 0.2.0 # Capability provided by this module. provides: - capability: llm.chat - version: 0.1.0 + version: 0.2.0 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -22,11 +22,26 @@ inputs: type: text description: en: | - Ollama-shaped /api/chat endpoint, e.g. - "http://localhost:11434/api/chat". + Chat endpoint URL matching the selected api: + ollama "http://localhost:11434/api/chat", + openai "https://.../v1/chat/completions" (vLLM etc.), + anthropic "https://api.anthropic.com/v1/messages". de: | - Ollama-kompatibler /api/chat-Endpunkt, z. B. - "http://localhost:11434/api/chat". + Chat-Endpunkt-URL passend zur gewählten api: + ollama "http://localhost:11434/api/chat", + openai "https://.../v1/chat/completions" (vLLM u. a.), + anthropic "https://api.anthropic.com/v1/messages". + api: + type: text + description: + en: | + Optional wire format: "ollama" (default), "openai" + (Chat Completions — OpenAI, vLLM, compatible servers), + or "anthropic" (Messages API). Empty = ollama. + de: | + Optionales Wire-Format: "ollama" (Default), "openai" + (Chat Completions — OpenAI, vLLM, kompatible Server) + oder "anthropic" (Messages API). Leer = ollama. model: type: text description: @@ -36,11 +51,15 @@ inputs: type: text description: en: | - Optional bearer token for cloud-hosted endpoints. - Empty string means no Authorization header is sent. + Optional API key. Sent as "Authorization: Bearer" for + ollama/openai and as "x-api-key" for anthropic. Empty + string means no auth header is sent. Never appears in + outputs, logs, or audit events. de: | - Optionaler Bearer-Token für Cloud-Endpunkte. - Leer = kein Authorization-Header gesendet. + Optionaler API-Key. Bei ollama/openai als + "Authorization: Bearer", bei anthropic als "x-api-key" + gesendet. Leer = kein Auth-Header. Erscheint nie in + Ausgaben, Logs oder Audit-Events. system_prompt: type: text description: diff --git a/src/lib.rs b/src/lib.rs index 54d729d..0ee9801 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,18 +1,23 @@ -//! `llm.chat` — generic Ollama-compatible LLM chat adapter. +//! `llm.chat` — generic LLM chat adapter. //! //! Sends a single user prompt (with optional system prompt) to an -//! Ollama `/api/chat` endpoint and returns the assistant text. -//! Reports `model_endpoint`, `model_name`, and `model_digest` as -//! separate outputs for audit consumers. +//! LLM endpoint and returns the assistant text. The wire format is +//! selected by the optional `api` input: `ollama` (default), +//! `openai` (Chat Completions — OpenAI, vLLM, and compatible +//! servers), or `anthropic` (Messages API). //! -//! v0.1.0 targets Ollama only. OpenAI / Anthropic adapters land -//! when a flow needs them. +//! Reports `model_endpoint`, `model_name`, and `model_digest` as +//! separate outputs for audit consumers. The digest is only +//! available from Ollama; the other APIs report it empty. +//! +//! The `api_key` input is placed into auth headers only — it never +//! appears in outputs, errors, or logs. mod llm; use chain_module_sdk::prelude::*; -#[fai_module] +#[chain_module] pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let prompt = inputs.require_text("prompt")?.to_string(); let endpoint = inputs.require_text("endpoint")?.to_string(); @@ -25,9 +30,13 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("system_prompt") .and_then(payload_text) .unwrap_or_default(); + let api_raw = inputs.get("api").and_then(payload_text).unwrap_or_default(); + let api = + crate::llm::Api::parse(&api_raw).map_err(|e| ModuleError::invalid_input(e.to_string()))?; let client = make_client(); let params = crate::llm::ChatParams { + api, endpoint: &endpoint, model: &model, api_key: &api_key, @@ -35,7 +44,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { prompt: &prompt, }; let result = crate::llm::chat_with_identity(&client, ¶ms) - .map_err(|e| llm_error_to_module_error(e, &endpoint, &model))?; + .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; Ok(Outputs::new() .with_text("response", result.response) @@ -53,29 +62,50 @@ fn payload_text(p: &Payload) -> Option { /// Turn a transport/protocol error into a message that names the /// likely cause and the fix, instead of a raw `ConnectionRefused`. -/// The endpoint is Ollama-shaped by default, so a connect failure -/// almost always means Ollama isn't running or the model isn't -/// pulled. -fn llm_error_to_module_error(e: crate::llm::LlmError, endpoint: &str, model: &str) -> ModuleError { - use crate::llm::LlmError; - match e { - LlmError::Http(detail) => ModuleError::internal(format!( +/// The hints are api-specific: an Ollama connect failure almost +/// always means Ollama isn't running or the model isn't pulled, +/// while cloud/vLLM failures are usually endpoint or key issues. +/// The message must never contain the api_key. +fn llm_error_to_module_error( + e: crate::llm::LlmError, + api: crate::llm::Api, + endpoint: &str, + model: &str, +) -> ModuleError { + use crate::llm::{Api, LlmError}; + match (api, e) { + (Api::Ollama, LlmError::Http(detail)) => ModuleError::internal(format!( "LLM endpoint {endpoint} not reachable ({detail}). Is Ollama running? \ Start it with `ollama serve`, then pull the model with `ollama pull {model}`. \ If the LLM runs elsewhere, set the `endpoint` input to its /api URL." )), - LlmError::Status(404) => ModuleError::internal(format!( + (Api::Openai, LlmError::Http(detail)) => ModuleError::internal(format!( + "LLM endpoint {endpoint} not reachable ({detail}). Expected an \ + OpenAI-compatible server (OpenAI, vLLM, ...) at a /v1/chat/completions URL." + )), + (Api::Anthropic, LlmError::Http(detail)) => ModuleError::internal(format!( + "LLM endpoint {endpoint} not reachable ({detail}). Expected the \ + Anthropic Messages API at a /v1/messages URL." + )), + (Api::Ollama, LlmError::Status(404)) => ModuleError::internal(format!( "LLM endpoint {endpoint} returned 404 for model '{model}' — the model is \ likely not pulled. Run `ollama pull {model}` (or check the model name)." )), - LlmError::Status(code) => ModuleError::internal(format!( + (_, LlmError::Status(401)) | (_, LlmError::Status(403)) => ModuleError::internal(format!( + "LLM endpoint {endpoint} rejected the request as unauthorized — check the \ + `api_key` input." + )), + (_, LlmError::Status(code)) => ModuleError::internal(format!( "LLM endpoint {endpoint} returned HTTP {code} for model '{model}'." )), - LlmError::Decode(detail) => ModuleError::internal(format!( - "LLM response from {endpoint} was not valid Ollama JSON: {detail}" + (_, LlmError::Decode(detail)) => ModuleError::internal(format!( + "LLM response from {endpoint} did not match the expected schema: {detail}" )), - LlmError::MissingInput(name) => ModuleError::invalid_input(format!( - "missing required input '{name}'" + (_, LlmError::MissingInput(name)) => { + ModuleError::invalid_input(format!("missing required input '{name}'")) + } + (_, LlmError::UnsupportedApi(raw)) => ModuleError::invalid_input(format!( + "unsupported api '{raw}' (expected: ollama, openai, anthropic)" )), } } @@ -100,7 +130,7 @@ impl crate::llm::LlmClient for HostStubClient { &self, _url: &str, _body: &str, - _api_key: &str, + _headers: &[(&'static str, String)], ) -> Result { Err(crate::llm::LlmError::Http( "LLM HTTP path is unavailable on the host build; only wasm32 supports outbound HTTP" @@ -118,14 +148,14 @@ impl crate::llm::LlmClient for WakiClient { &self, url: &str, body: &str, - api_key: &str, + headers: &[(&'static str, String)], ) -> Result { let mut request = waki::Client::new() .post(url) .header("Content-Type", "application/json") .body(body.to_string()); - if !api_key.is_empty() { - request = request.header("Authorization", &format!("Bearer {api_key}")); + for (name, value) in headers { + request = request.header(*name, value); } let response = request .send() diff --git a/src/llm.rs b/src/llm.rs index e09707f..90565f5 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -1,35 +1,84 @@ -//! Ollama-shaped chat client. +//! Multi-API chat client: Ollama (default), OpenAI, Anthropic. //! -//! v0.1.0 targets the Ollama `/api/chat` endpoint. OpenAI- and -//! Anthropic-compatible adapters are deliberately deferred — each -//! has its own request/response shape that warrants its own crate -//! (or at least its own module here) once a flow needs it. +//! The wire format is selected via the optional `api` input: +//! +//! - `ollama` (default) — Ollama `/api/chat`; unchanged v0.1.0 +//! behavior including the best-effort model-digest probe. +//! - `openai` — OpenAI Chat Completions wire format +//! (`/v1/chat/completions`), the de-facto standard implemented +//! by vLLM and most self-hosted inference servers. Bearer auth. +//! - `anthropic` — Anthropic Messages API (`/v1/messages`), +//! `x-api-key` + `anthropic-version` headers, top-level +//! `system` field, mandatory `max_tokens`. +//! +//! No streaming, no tool calls — one prompt in, one completion out. //! //! All HTTP I/O lives behind a `LlmClient` trait so unit tests can -//! exercise prompt building, response parsing, and the digest +//! exercise request building, response parsing, and the digest //! probe on the host without making real network calls. use serde::Serialize; +/// Anthropic's Messages API requires `max_tokens`. This default is +/// large enough for document-processing completions while staying +/// well below every current model's output cap. +const ANTHROPIC_DEFAULT_MAX_TOKENS: u32 = 4096; + +/// Pinned Messages API version header. Anthropic keeps old +/// versions working; bump deliberately, never implicitly. +const ANTHROPIC_VERSION: &str = "2023-06-01"; + #[allow(dead_code)] #[derive(Debug, thiserror::Error)] pub enum LlmError { #[error("missing required input '{0}'")] MissingInput(&'static str), + #[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")] + UnsupportedApi(String), #[error("http error: {0}")] Http(String), #[error("non-success status: {0}")] Status(u16), - #[error("response body could not be parsed as Ollama schema: {0}")] + #[error("response body could not be parsed as the expected schema: {0}")] Decode(String), } +/// Which wire format to speak. Selected by the `api` input. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Api { + Ollama, + Openai, + Anthropic, +} + +impl Api { + /// Parse the `api` input. Empty string means the input was not + /// provided and falls back to Ollama (v0.1.0 behavior). + pub fn parse(raw: &str) -> Result { + match raw.trim().to_ascii_lowercase().as_str() { + "" | "ollama" => Ok(Api::Ollama), + "openai" => Ok(Api::Openai), + "anthropic" => Ok(Api::Anthropic), + other => Err(LlmError::UnsupportedApi(other.to_string())), + } + } +} + pub trait LlmClient { - fn post_json(&self, url: &str, body: &str, api_key: &str) -> Result; + /// POST `body` as JSON to `url` with the given extra headers. + /// `Content-Type: application/json` is implied. Header VALUES + /// may carry credentials — implementations must never log them. + fn post_json( + &self, + url: &str, + body: &str, + headers: &[(&'static str, String)], + ) -> Result; } #[derive(Debug, Clone)] pub struct ChatParams<'a> { + pub api: Api, pub endpoint: &'a str, pub model: &'a str, pub api_key: &'a str, @@ -37,8 +86,29 @@ pub struct ChatParams<'a> { pub prompt: &'a str, } +/// Auth/protocol headers for the selected API. The api_key is only +/// ever placed into a header value here; it must not appear in any +/// error, output, or log line. +pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> { + let mut headers = Vec::with_capacity(2); + match p.api { + Api::Ollama | Api::Openai => { + if !p.api_key.is_empty() { + headers.push(("Authorization", format!("Bearer {}", p.api_key))); + } + } + Api::Anthropic => { + if !p.api_key.is_empty() { + headers.push(("x-api-key", p.api_key.to_string())); + } + headers.push(("anthropic-version", ANTHROPIC_VERSION.to_string())); + } + } + headers +} + #[derive(Serialize)] -struct OllamaMessage<'a> { +struct ChatMessage<'a> { role: &'a str, content: &'a str, } @@ -46,33 +116,86 @@ struct OllamaMessage<'a> { #[derive(Serialize)] struct OllamaRequest<'a> { model: &'a str, - messages: Vec>, + messages: Vec>, stream: bool, } -/// Build the request body for an Ollama `/api/chat` invocation. -/// The system prompt is omitted from the messages list when empty -/// so a deployment can use the model's built-in system prompt. -pub fn build_ollama_body(p: &ChatParams) -> String { +#[derive(Serialize)] +struct OpenAiRequest<'a> { + model: &'a str, + messages: Vec>, + stream: bool, +} + +#[derive(Serialize)] +struct AnthropicRequest<'a> { + model: &'a str, + max_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + system: Option<&'a str>, + messages: Vec>, +} + +/// System-then-user message list shared by the Ollama and OpenAI +/// wire formats. The system message is omitted when empty so a +/// deployment can use the model's built-in system prompt. +fn build_messages<'a>(p: &ChatParams<'a>) -> Vec> { let mut messages = Vec::with_capacity(2); if !p.system_prompt.is_empty() { - messages.push(OllamaMessage { + messages.push(ChatMessage { role: "system", content: p.system_prompt, }); } - messages.push(OllamaMessage { + messages.push(ChatMessage { role: "user", content: p.prompt, }); + messages +} + +/// Build the request body for an Ollama `/api/chat` invocation. +pub fn build_ollama_body(p: &ChatParams) -> String { let req = OllamaRequest { model: p.model, - messages, + messages: build_messages(p), stream: false, }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } +/// Build the request body for an OpenAI-compatible +/// `/v1/chat/completions` invocation (OpenAI, vLLM, most +/// self-hosted inference servers). +pub fn build_openai_body(p: &ChatParams) -> String { + let req = OpenAiRequest { + model: p.model, + messages: build_messages(p), + stream: false, + }; + serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) +} + +/// Build the request body for an Anthropic `/v1/messages` +/// invocation. `system` is a top-level field (not a message); +/// `max_tokens` is mandatory in the Messages API. +pub fn build_anthropic_body(p: &ChatParams) -> String { + let req = AnthropicRequest { + model: p.model, + max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, + system: if p.system_prompt.is_empty() { + None + } else { + Some(p.system_prompt) + }, + messages: vec![ChatMessage { + role: "user", + content: p.prompt, + }], + }; + serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) +} + /// Extract the assistant's message text from an Ollama /// /api/chat response. pub fn extract_ollama_content(body: &str) -> Result { @@ -85,16 +208,44 @@ pub fn extract_ollama_content(body: &str) -> Result { .ok_or_else(|| LlmError::Decode("missing message.content".into())) } +/// Extract the assistant text from an OpenAI Chat Completions +/// response: `choices[0].message.content`. +pub fn extract_openai_content(body: &str) -> Result { + let v: serde_json::Value = + serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; + v.get("choices") + .and_then(|c| c.get(0)) + .and_then(|c| c.get("message")) + .and_then(|m| m.get("content")) + .and_then(|c| c.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| LlmError::Decode("missing choices[0].message.content".into())) +} + +/// Extract the assistant text from an Anthropic Messages +/// response: `content[0].text`. +pub fn extract_anthropic_content(body: &str) -> Result { + let v: serde_json::Value = + serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; + v.get("content") + .and_then(|c| c.get(0)) + .and_then(|c| c.get("text")) + .and_then(|t| t.as_str()) + .map(|s| s.to_string()) + .ok_or_else(|| LlmError::Decode("missing content[0].text".into())) +} + #[derive(Debug, Clone)] pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, } -/// Run an Ollama chat call AND probe the model digest. Probe is -/// best-effort — non-Ollama endpoints (no `/api/chat` suffix) and -/// transient failures yield `model_digest = None` rather than -/// failing the whole call. +/// Run one chat call in the selected wire format AND, for Ollama, +/// probe the model digest. The probe is best-effort — non-Ollama +/// APIs expose no digest endpoint, so `model_digest = None` there +/// (documented empty, never invented); transient probe failures on +/// Ollama also yield `None` rather than failing the whole call. pub fn chat_with_identity( client: &C, p: &ChatParams, @@ -108,10 +259,22 @@ pub fn chat_with_identity( if p.prompt.is_empty() { return Err(LlmError::MissingInput("prompt")); } - let body = build_ollama_body(p); - let response_body = client.post_json(p.endpoint, &body, p.api_key)?; - let response = extract_ollama_content(&response_body)?; - let model_digest = probe_model_digest(client, p); + let body = match p.api { + Api::Ollama => build_ollama_body(p), + Api::Openai => build_openai_body(p), + Api::Anthropic => build_anthropic_body(p), + }; + let headers = build_headers(p); + let response_body = client.post_json(p.endpoint, &body, &headers)?; + let response = match p.api { + Api::Ollama => extract_ollama_content(&response_body)?, + Api::Openai => extract_openai_content(&response_body)?, + Api::Anthropic => extract_anthropic_content(&response_body)?, + }; + let model_digest = match p.api { + Api::Ollama => probe_model_digest(client, p), + Api::Openai | Api::Anthropic => None, + }; Ok(ChatWithIdentity { response, model_digest, @@ -121,7 +284,8 @@ pub fn chat_with_identity( fn probe_model_digest(client: &C, p: &ChatParams) -> Option { let show_url = derive_show_url(p.endpoint)?; let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?; - let response = client.post_json(&show_url, &body, p.api_key).ok()?; + let headers = build_headers(p); + let response = client.post_json(&show_url, &body, &headers).ok()?; extract_show_digest(&response) } @@ -160,20 +324,39 @@ mod tests { use super::*; use std::cell::RefCell; + /// (url, body, headers) of one recorded call. + type RecordedCall = (String, String, Vec<(String, String)>); + struct MockClient { responses: RefCell>>, + /// Every call, in call order. + calls: RefCell>, } impl MockClient { fn new(responses: Vec>) -> Self { Self { responses: RefCell::new(responses), + calls: RefCell::new(Vec::new()), } } } impl LlmClient for MockClient { - fn post_json(&self, _url: &str, _body: &str, _api_key: &str) -> Result { + fn post_json( + &self, + url: &str, + body: &str, + headers: &[(&'static str, String)], + ) -> Result { + self.calls.borrow_mut().push(( + url.to_string(), + body.to_string(), + headers + .iter() + .map(|(k, v)| (k.to_string(), v.clone())) + .collect(), + )); self.responses .borrow_mut() .pop() @@ -181,14 +364,42 @@ mod tests { } } - #[test] - fn ollama_body_includes_system_when_provided() { - let p = ChatParams { + fn params<'a>(api: Api) -> ChatParams<'a> { + ChatParams { + api, endpoint: "http://x/api/chat", model: "qwen", api_key: "", - system_prompt: "be helpful", + system_prompt: "", prompt: "hello", + } + } + + // ---------------- api input parsing ---------------- + + #[test] + fn api_parse_defaults_to_ollama() { + assert_eq!(Api::parse("").unwrap(), Api::Ollama); + assert_eq!(Api::parse("ollama").unwrap(), Api::Ollama); + assert_eq!(Api::parse(" OpenAI ").unwrap(), Api::Openai); + assert_eq!(Api::parse("anthropic").unwrap(), Api::Anthropic); + } + + #[test] + fn api_parse_rejects_unknown() { + assert!(matches!( + Api::parse("gemini"), + Err(LlmError::UnsupportedApi(_)) + )); + } + + // ---------------- Ollama wire format (regression) ---------------- + + #[test] + fn ollama_body_includes_system_when_provided() { + let p = ChatParams { + system_prompt: "be helpful", + ..params(Api::Ollama) }; let body = build_ollama_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); @@ -198,18 +409,12 @@ mod tests { assert_eq!(messages[0]["content"], "be helpful"); assert_eq!(messages[1]["role"], "user"); assert_eq!(messages[1]["content"], "hello"); + assert_eq!(v["stream"], false); } #[test] fn ollama_body_omits_system_when_empty() { - let p = ChatParams { - endpoint: "http://x/api/chat", - model: "qwen", - api_key: "", - system_prompt: "", - prompt: "hello", - }; - let body = build_ollama_body(&p); + let body = build_ollama_body(¶ms(Api::Ollama)); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 1); @@ -231,6 +436,208 @@ mod tests { )); } + #[test] + fn ollama_bearer_header_only_with_key() { + assert!(build_headers(¶ms(Api::Ollama)).is_empty()); + let p = ChatParams { + api_key: "sk-test", + ..params(Api::Ollama) + }; + assert_eq!( + build_headers(&p), + vec![("Authorization", "Bearer sk-test".to_string())] + ); + } + + // ---------------- OpenAI wire format ---------------- + + #[test] + fn openai_body_matches_chat_completions_format() { + let p = ChatParams { + api: Api::Openai, + endpoint: "http://vllm:8000/v1/chat/completions", + model: "meta-llama/Llama-3.1-8B-Instruct", + api_key: "sk-x", + system_prompt: "be terse", + prompt: "hello", + }; + let body = build_openai_body(&p); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct"); + assert_eq!(v["stream"], false); + let messages = v["messages"].as_array().unwrap(); + assert_eq!(messages.len(), 2); + assert_eq!(messages[0]["role"], "system"); + assert_eq!(messages[0]["content"], "be terse"); + assert_eq!(messages[1]["role"], "user"); + assert_eq!(messages[1]["content"], "hello"); + // No Anthropic-only fields leak into the OpenAI body. + assert!(v.get("max_tokens").is_none()); + assert!(v.get("system").is_none()); + } + + #[test] + fn openai_response_parses_fixture() { + // Shape as returned by OpenAI / vLLM /v1/chat/completions. + let body = r#"{ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1719000000, + "model": "meta-llama/Llama-3.1-8B-Instruct", + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "Hi from vLLM"}, + "finish_reason": "stop" + }], + "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13} + }"#; + assert_eq!(extract_openai_content(body).unwrap(), "Hi from vLLM"); + } + + #[test] + fn openai_response_errors_on_missing_choices() { + assert!(matches!( + extract_openai_content(r#"{"object":"chat.completion","choices":[]}"#), + Err(LlmError::Decode(_)) + )); + assert!(matches!( + extract_openai_content(r#"{"error":{"message":"invalid key"}}"#), + Err(LlmError::Decode(_)) + )); + } + + #[test] + fn openai_uses_bearer_auth() { + let p = ChatParams { + api_key: "sk-test", + ..params(Api::Openai) + }; + assert_eq!( + build_headers(&p), + vec![("Authorization", "Bearer sk-test".to_string())] + ); + } + + #[test] + fn openai_chat_returns_content_and_no_digest() { + let canned = r#"{"choices":[{"message":{"role":"assistant","content":"ok"}}]}"#; + let client = MockClient::new(vec![Ok(canned.to_string())]); + let p = ChatParams { + api: Api::Openai, + endpoint: "http://vllm:8000/v1/chat/completions", + model: "m", + api_key: "sk", + system_prompt: "", + prompt: "hi", + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.response, "ok"); + // No digest API on OpenAI-compatible endpoints — documented + // empty, and exactly one HTTP call (no probe). + assert_eq!(result.model_digest, None); + assert_eq!(client.calls.borrow().len(), 1); + } + + // ---------------- Anthropic wire format ---------------- + + #[test] + fn anthropic_body_matches_messages_format() { + let p = ChatParams { + api: Api::Anthropic, + endpoint: "https://api.anthropic.com/v1/messages", + model: "claude-fable-5", + api_key: "sk-ant", + system_prompt: "be terse", + prompt: "hello", + }; + let body = build_anthropic_body(&p); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["model"], "claude-fable-5"); + // max_tokens is mandatory in the Messages API. + assert_eq!(v["max_tokens"], 4096); + // system is a top-level field, never a message. + assert_eq!(v["system"], "be terse"); + let messages = v["messages"].as_array().unwrap(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0]["role"], "user"); + assert_eq!(messages[0]["content"], "hello"); + // No OpenAI/Ollama-only fields leak in. + assert!(v.get("stream").is_none()); + } + + #[test] + fn anthropic_body_omits_system_when_empty() { + let body = build_anthropic_body(¶ms(Api::Anthropic)); + let v: serde_json::Value = serde_json::from_str(&body).unwrap(); + assert!(v.get("system").is_none()); + assert_eq!(v["max_tokens"], 4096); + } + + #[test] + fn anthropic_response_parses_fixture() { + // Shape as returned by the Anthropic Messages API. + let body = r#"{ + "id": "msg_01XFDUDYJgAACzvnptvVoYEL", + "type": "message", + "role": "assistant", + "model": "claude-fable-5", + "content": [{"type": "text", "text": "Hi from Claude"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5} + }"#; + assert_eq!(extract_anthropic_content(body).unwrap(), "Hi from Claude"); + } + + #[test] + fn anthropic_response_errors_on_missing_content() { + assert!(matches!( + extract_anthropic_content(r#"{"type":"message","content":[]}"#), + Err(LlmError::Decode(_)) + )); + assert!(matches!( + extract_anthropic_content(r#"{"type":"error","error":{"message":"x"}}"#), + Err(LlmError::Decode(_)) + )); + } + + #[test] + fn anthropic_uses_x_api_key_and_version_headers() { + let p = ChatParams { + api_key: "sk-ant-test", + ..params(Api::Anthropic) + }; + let headers = build_headers(&p); + assert_eq!( + headers, + vec![ + ("x-api-key", "sk-ant-test".to_string()), + ("anthropic-version", "2023-06-01".to_string()), + ] + ); + // Never Bearer auth on the Anthropic path. + assert!(headers.iter().all(|(k, _)| *k != "Authorization")); + } + + #[test] + fn anthropic_chat_returns_content_and_no_digest() { + let canned = r#"{"content":[{"type":"text","text":"ok"}]}"#; + let client = MockClient::new(vec![Ok(canned.to_string())]); + let p = ChatParams { + api: Api::Anthropic, + endpoint: "https://api.anthropic.com/v1/messages", + model: "claude-fable-5", + api_key: "sk-ant", + system_prompt: "", + prompt: "hi", + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.response, "ok"); + assert_eq!(result.model_digest, None); + assert_eq!(client.calls.borrow().len(), 1); + } + + // ---------------- digest probe (Ollama only) ---------------- + #[test] fn derive_show_url_swaps_chat_suffix() { assert_eq!( @@ -263,10 +670,7 @@ mod tests { let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", - model: "qwen", - api_key: "", - system_prompt: "", - prompt: "hi", + ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "the response"); @@ -279,10 +683,7 @@ mod tests { let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", - model: "qwen", - api_key: "", - system_prompt: "", - prompt: "hi", + ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -290,30 +691,26 @@ mod tests { } #[test] - fn chat_with_identity_skips_probe_for_non_ollama() { + fn chat_with_identity_skips_probe_for_non_ollama_url() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); let client = MockClient::new(vec![Ok(canned_chat)]); let p = ChatParams { - endpoint: "https://api.openai.com/v1/chat/completions", - model: "gpt", - api_key: "k", - system_prompt: "", - prompt: "hi", + endpoint: "https://example.com/proxy/chat", + ..params(Api::Ollama) }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); assert_eq!(result.model_digest, None); } + // ---------------- input guards ---------------- + #[test] fn empty_prompt_is_rejected() { let client = MockClient::new(vec![]); let p = ChatParams { - endpoint: "http://x/api/chat", - model: "x", - api_key: "", - system_prompt: "", prompt: "", + ..params(Api::Ollama) }; assert!(matches!( chat_with_identity(&client, &p),