diff --git a/Cargo.lock b/Cargo.lock index 462390b..dcff588 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,8 +40,8 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chain-module-sdk" -version = "0.3.0" -source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?tag=v0.3.0#a4415ada7e76aac5e3aabf3e4c4311ffc348d8d0" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" dependencies = [ "chain-module-sdk-macros", "serde", @@ -52,8 +52,8 @@ dependencies = [ [[package]] name = "chain-module-sdk-macros" -version = "0.3.0" -source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?tag=v0.3.0#a4415ada7e76aac5e3aabf3e4c4311ffc348d8d0" +version = "0.1.2" +source = "git+https://git.flemming.ai/fai/chain-module-sdk-rust.git?branch=main#98e98e9371f7409560a1ef08bc0923d9a2506449" dependencies = [ "proc-macro2", "quote", @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.2.0" +version = "0.1.0" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6a761ca..72db65b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.2.0" +version = "0.1.0" edition = "2024" license = "Apache-2.0" publish = false @@ -17,7 +17,7 @@ description = "F∆I module providing llm.chat" crate-type = ["cdylib", "rlib"] [dependencies] -chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", tag = "v0.3.0" } +chain-module-sdk = { git = "https://git.flemming.ai/fai/chain-module-sdk-rust.git", branch = "main" } serde = { version = "1", features = ["derive"] } serde_json = "1" thiserror = "2" diff --git a/MODULE.de.md b/MODULE.de.md index ab71365..4d12be6 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -1,35 +1,23 @@ # llm.chat -Generischer LLM-Chat-Adapter. Das Baustein-Modul für jeden -Flow, der eine LLM-Einzelantwort braucht — klassifizieren, -Felder extrahieren, umformulieren, entscheiden — ohne einen -eigenen HTTP-Client zu bauen. Spricht drei Wire-Formate, -wählbar über die optionale Eingabe `api`: Ollama (Default), -OpenAI Chat Completions (OpenAI, vLLM, kompatible Server) und -die Anthropic Messages API. +Generischer Ollama-kompatibler LLM-Chat-Adapter. Das +Baustein-Modul für jeden Flow, der eine LLM-Einzelantwort +braucht — klassifizieren, Felder extrahieren, umformulieren, +entscheiden — ohne einen eigenen HTTP-Client zu bauen. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.1.0` ## Eingaben | Name | Typ | Beschreibung | | ---------------- | ---- | ------------------------------------------------------------------------- | | `prompt` | text | Der User-Prompt. | -| `endpoint` | text | Chat-Endpunkt-URL passend zur gewählten `api` (siehe unten). | -| `model` | text | Modell-ID (z. B. `qwen2.5:14b`, `claude-fable-5`). | -| `api_key` | text | Optionaler API-Key. Erscheint nie in Ausgaben, Logs oder Events. | +| `endpoint` | text | Ollama-förmiger `/api/chat`-URL (z.B. `http://localhost:11434/api/chat`). | +| `model` | text | Modell-ID (z.B. `qwen2.5:14b`, `llama3.1:8b`). | +| `api_key` | text | Optionaler Bearer-Token für Cloud-Endpunkte. | | `system_prompt` | text | Optionale System-Nachricht. Leer = Default des Modells. | -| `api` | text | Optionales Wire-Format: `ollama` (Default), `openai`, `anthropic`. | - -### Wire-Formate (`api`) - -| `api` | Endpunkt-Form | Auth-Header | Hinweise | -| ----------- | ----------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | -| `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | Verhalten wie v0.1.0, unverändert. Modell-Digest-Probe via `/api/show`. | -| `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI-Chat-Completions-Format — sprechen auch vLLM und die meisten selbst gehosteten Inferenz-Server. | -| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` ist Top-Level-Feld; `max_tokens` ist Pflicht und steht fest auf 4096. | ## Ausgaben @@ -38,7 +26,7 @@ die Anthropic Messages API. | `response` | text | Die Antwort des Assistants als Plain-Text. | | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | -| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | +| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei Cloud-APIs. | Zusammen beantworten die drei `model_*`-Ausgaben die Audit- Frage: „Welches genaue Modell hat diese Antwort erzeugt?" — @@ -55,8 +43,8 @@ permissions: - "net: api.anthropic.com" ``` -Loopback (lokales Ollama / vLLM) per Default. Cloud-Endpunkte -brauchen einen Operator-Policy-Override in +Loopback (lokales Ollama) per Default. Cloud-Endpunkte brauchen +einen Operator-Policy-Override in `~/.chain/config.yaml#security.max_permissions`. ## Warum dieses Modul statt Inline-HTTP @@ -73,24 +61,23 @@ Drei Gründe: `net:`-Liste. Ein hausgemachtes HTTP-Modul würde seine Endpunkte entweder verstecken oder bei jeder Installation eine frische Review-Oberfläche schaffen. -3. **Endpunkt-Portabilität.** Endpunkt, Modell und Wire-Format - sind Flow-Inputs, keine Compile-Time-Konstanten. Derselbe - Flow läuft in der Entwicklung gegen `localhost:11434` und - in Produktion gegen ein vLLM-Cluster — nur die Eingaben - wechseln. +3. **Endpunkt-Portabilität.** Endpunkt + Modell sind Flow- + Inputs, keine Compile-Time-Konstanten. Derselbe Flow läuft + in der Entwicklung gegen `localhost:11434` und in Produktion + gegen ein Inferenz-Cluster — nur die Eingabe wechselt. -## Grenzen in v0.2.0 +## Grenzen in v0.1.0 - Kein Streaming. Die ganze Antwort wird gepuffert, bevor der Output-Step feuert. - Keine Tool-Call- / Function-Call-Oberfläche. Ein Flow, der Tool-Use braucht, kombiniert mehrere `llm.chat`-Schritte mit Prompt-Engineering oder nutzt MCP via Bridge. -- Anthropic-`max_tokens` steht fest auf 4096 (die API verlangt - das Feld; eine konfigurierbare Eingabe kommt, sobald ein - Flow sie braucht). +- Cloud-Provider-Adapter für OpenAI und Anthropic kommen erst, + wenn ein Flow sie braucht. Heute ist das Ollama-Wire-Format + das einzige Ziel. -## Beispiel-Flows +## Beispiel-Flow ```yaml name: classify-incoming @@ -111,33 +98,6 @@ outputs: audit_model: $classify.model_digest ``` -Gegen einen vLLM-Server (OpenAI-kompatibel): - -```yaml -steps: - - id: classify - use: llm.chat@^0 - with: - api: "openai" - endpoint: "http://inference.internal:8000/v1/chat/completions" - model: "meta-llama/Llama-3.1-8B-Instruct" - prompt: $inputs.text -``` - -Gegen die Anthropic Messages API: - -```yaml -steps: - - id: classify - use: llm.chat@^0 - with: - api: "anthropic" - endpoint: "https://api.anthropic.com/v1/messages" - model: "claude-fable-5" - api_key: $inputs.anthropic_key - prompt: $inputs.text -``` - ## Build ```bash diff --git a/MODULE.md b/MODULE.md index cb8c167..49460ad 100644 --- a/MODULE.md +++ b/MODULE.md @@ -1,35 +1,23 @@ # llm.chat -Generic LLM chat adapter. The building-block module any flow -uses when it needs a one-shot LLM completion — classify, -extract-fields, rewrite, decide — instead of rolling its own -HTTP client. Speaks three wire formats, selected by the -optional `api` input: Ollama (default), OpenAI Chat -Completions (OpenAI, vLLM, compatible servers), and the -Anthropic Messages API. +Generic Ollama-compatible LLM chat adapter. The building-block +module any flow uses when it needs a one-shot LLM completion — +classify, extract-fields, rewrite, decide — instead of rolling +its own HTTP client. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.1.0` ## Inputs | Name | Type | Description | | ---------------- | ---- | --------------------------------------------------------------- | | `prompt` | text | The user-facing prompt. | -| `endpoint` | text | Chat endpoint URL matching the selected `api` (see below). | -| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `claude-fable-5`). | -| `api_key` | text | Optional API key. Never appears in outputs, logs, or events. | +| `endpoint` | text | Ollama-shaped `/api/chat` URL (e.g. `http://localhost:11434/api/chat`). | +| `model` | text | Model identifier (e.g. `qwen2.5:14b`, `llama3.1:8b`). | +| `api_key` | text | Optional bearer token for cloud-hosted endpoints. | | `system_prompt` | text | Optional system message. Empty = use the model's default. | -| `api` | text | Optional wire format: `ollama` (default), `openai`, `anthropic`. | - -### Wire formats (`api`) - -| `api` | Endpoint shape | Auth header | Notes | -| ----------- | --------------------------------------------- | ---------------------------------- | ------------------------------------------------------------ | -| `ollama` | `http://localhost:11434/api/chat` | `Authorization: Bearer` (optional) | v0.1.0 behavior, unchanged. Model-digest probe via `/api/show`. | -| `openai` | `https://.../v1/chat/completions` | `Authorization: Bearer` | OpenAI Chat Completions format — also spoken by vLLM and most self-hosted inference servers. | -| `anthropic` | `https://api.anthropic.com/v1/messages` | `x-api-key` + `anthropic-version` | Messages API: `system` is a top-level field; `max_tokens` is mandatory and defaults to 4096. | ## Outputs @@ -38,7 +26,7 @@ Anthropic Messages API. | `response` | text | The assistant's plain-text reply. | | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | -| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | +| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for cloud APIs. | Together the three `model_*` outputs answer the audit question: "which exact model produced this response?" — that @@ -55,7 +43,7 @@ permissions: - "net: api.anthropic.com" ``` -Loopback (local Ollama / vLLM) by default. Cloud endpoints require an +Loopback (local Ollama) by default. Cloud endpoints require an operator-policy override in `~/.chain/config.yaml#security.max_permissions`. ## Why this module instead of inline HTTP @@ -71,22 +59,23 @@ Three reasons: the installed modules + its declared `net:` list. A home-grown HTTP module would either hide its endpoints or be a fresh review surface every time. -3. **Endpoint portability.** The endpoint, model, and wire - format are flow inputs, not compile-time constants. The - same flow runs against `localhost:11434` in dev and a - production vLLM cluster in prod just by swapping inputs. +3. **Endpoint portability.** The endpoint + model are flow + inputs, not compile-time constants. The same flow runs + against `localhost:11434` in dev and a production + inference cluster in prod just by swapping the input. -## Limits in v0.2.0 +## Limits in v0.1.0 - No streaming. The whole reply is buffered before the output step fires. - No tool-call / function-call surface. A flow needing tool use composes multiple `llm.chat` steps with prompt engineering, or uses MCP via the bridge. -- Anthropic `max_tokens` is fixed at 4096 (the API requires - the field; a configurable input lands when a flow needs it). +- Cloud-provider adapters for OpenAI and Anthropic are + deferred until a flow actually needs them. Today the + Ollama wire-format is the only target. -## Example flows +## Example flow ```yaml name: classify-incoming @@ -107,33 +96,6 @@ outputs: audit_model: $classify.model_digest ``` -Against a vLLM (OpenAI-compatible) server: - -```yaml -steps: - - id: classify - use: llm.chat@^0 - with: - api: "openai" - endpoint: "http://inference.internal:8000/v1/chat/completions" - model: "meta-llama/Llama-3.1-8B-Instruct" - prompt: $inputs.text -``` - -Against the Anthropic Messages API: - -```yaml -steps: - - id: classify - use: llm.chat@^0 - with: - api: "anthropic" - endpoint: "https://api.anthropic.com/v1/messages" - model: "claude-fable-5" - api_key: $inputs.anthropic_key - prompt: $inputs.text -``` - ## Build ```bash diff --git a/README.md b/README.md index 5e55757..275de8d 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ # llm-chat F∆I module providing the `llm.chat` capability — a generic -LLM chat adapter speaking Ollama (default), OpenAI Chat -Completions (OpenAI, vLLM, compatible servers), and the -Anthropic Messages API, selected via the optional `api` input. +Ollama-compatible LLM chat adapter. ## Capability | Field | Value | |-------|-------| -| Capability | `llm.chat@0.2.0` | -| Inputs | `prompt: text`, `endpoint: text`, `model: text`, `api_key: text` (opt), `system_prompt: text` (opt), `api: text` (opt: `ollama`\|`openai`\|`anthropic`) | +| Capability | `llm.chat@0.1.0` | +| Inputs | `prompt: text`, `endpoint: text`, `model: text`, `api_key: text` (opt), `system_prompt: text` (opt) | | Outputs | `response: text`, `model_endpoint: text`, `model_name: text`, `model_digest: text` | | Permissions | `net: localhost`, `net: 127.0.0.1`, `net: api.openai.com`, `net: api.anthropic.com` | | Status (in store index) | `alpha` | @@ -43,7 +41,7 @@ cargo build --release --target wasm32-wasip2 ## Test ```bash -cargo test # wire-format unit tests, all host-side +cargo test # 11 tests, all host-side cargo build --release --target wasm32-wasip2 ``` diff --git a/module.wasm b/module.wasm index 353c547..a531ccd 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 4472d9d..503111a 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.2.0 +version: 0.1.0 # Capability provided by this module. provides: - capability: llm.chat - version: 0.2.0 + version: 0.1.0 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -22,26 +22,11 @@ inputs: type: text description: en: | - Chat endpoint URL matching the selected api: - ollama "http://localhost:11434/api/chat", - openai "https://.../v1/chat/completions" (vLLM etc.), - anthropic "https://api.anthropic.com/v1/messages". + Ollama-shaped /api/chat endpoint, e.g. + "http://localhost:11434/api/chat". de: | - Chat-Endpunkt-URL passend zur gewählten api: - ollama "http://localhost:11434/api/chat", - openai "https://.../v1/chat/completions" (vLLM u. a.), - anthropic "https://api.anthropic.com/v1/messages". - api: - type: text - description: - en: | - Optional wire format: "ollama" (default), "openai" - (Chat Completions — OpenAI, vLLM, compatible servers), - or "anthropic" (Messages API). Empty = ollama. - de: | - Optionales Wire-Format: "ollama" (Default), "openai" - (Chat Completions — OpenAI, vLLM, kompatible Server) - oder "anthropic" (Messages API). Leer = ollama. + Ollama-kompatibler /api/chat-Endpunkt, z. B. + "http://localhost:11434/api/chat". model: type: text description: @@ -51,15 +36,11 @@ inputs: type: text description: en: | - Optional API key. Sent as "Authorization: Bearer" for - ollama/openai and as "x-api-key" for anthropic. Empty - string means no auth header is sent. Never appears in - outputs, logs, or audit events. + Optional bearer token for cloud-hosted endpoints. + Empty string means no Authorization header is sent. de: | - Optionaler API-Key. Bei ollama/openai als - "Authorization: Bearer", bei anthropic als "x-api-key" - gesendet. Leer = kein Auth-Header. Erscheint nie in - Ausgaben, Logs oder Audit-Events. + Optionaler Bearer-Token für Cloud-Endpunkte. + Leer = kein Authorization-Header gesendet. system_prompt: type: text description: diff --git a/src/lib.rs b/src/lib.rs index 0ee9801..54d729d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,23 +1,18 @@ -//! `llm.chat` — generic LLM chat adapter. +//! `llm.chat` — generic Ollama-compatible LLM chat adapter. //! //! Sends a single user prompt (with optional system prompt) to an -//! LLM endpoint and returns the assistant text. The wire format is -//! selected by the optional `api` input: `ollama` (default), -//! `openai` (Chat Completions — OpenAI, vLLM, and compatible -//! servers), or `anthropic` (Messages API). -//! +//! Ollama `/api/chat` endpoint and returns the assistant text. //! Reports `model_endpoint`, `model_name`, and `model_digest` as -//! separate outputs for audit consumers. The digest is only -//! available from Ollama; the other APIs report it empty. +//! separate outputs for audit consumers. //! -//! The `api_key` input is placed into auth headers only — it never -//! appears in outputs, errors, or logs. +//! v0.1.0 targets Ollama only. OpenAI / Anthropic adapters land +//! when a flow needs them. mod llm; use chain_module_sdk::prelude::*; -#[chain_module] +#[fai_module] pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let prompt = inputs.require_text("prompt")?.to_string(); let endpoint = inputs.require_text("endpoint")?.to_string(); @@ -30,13 +25,9 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .get("system_prompt") .and_then(payload_text) .unwrap_or_default(); - let api_raw = inputs.get("api").and_then(payload_text).unwrap_or_default(); - let api = - crate::llm::Api::parse(&api_raw).map_err(|e| ModuleError::invalid_input(e.to_string()))?; let client = make_client(); let params = crate::llm::ChatParams { - api, endpoint: &endpoint, model: &model, api_key: &api_key, @@ -44,7 +35,7 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { prompt: &prompt, }; let result = crate::llm::chat_with_identity(&client, ¶ms) - .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; + .map_err(|e| llm_error_to_module_error(e, &endpoint, &model))?; Ok(Outputs::new() .with_text("response", result.response) @@ -62,50 +53,29 @@ fn payload_text(p: &Payload) -> Option { /// Turn a transport/protocol error into a message that names the /// likely cause and the fix, instead of a raw `ConnectionRefused`. -/// The hints are api-specific: an Ollama connect failure almost -/// always means Ollama isn't running or the model isn't pulled, -/// while cloud/vLLM failures are usually endpoint or key issues. -/// The message must never contain the api_key. -fn llm_error_to_module_error( - e: crate::llm::LlmError, - api: crate::llm::Api, - endpoint: &str, - model: &str, -) -> ModuleError { - use crate::llm::{Api, LlmError}; - match (api, e) { - (Api::Ollama, LlmError::Http(detail)) => ModuleError::internal(format!( +/// The endpoint is Ollama-shaped by default, so a connect failure +/// almost always means Ollama isn't running or the model isn't +/// pulled. +fn llm_error_to_module_error(e: crate::llm::LlmError, endpoint: &str, model: &str) -> ModuleError { + use crate::llm::LlmError; + match e { + LlmError::Http(detail) => ModuleError::internal(format!( "LLM endpoint {endpoint} not reachable ({detail}). Is Ollama running? \ Start it with `ollama serve`, then pull the model with `ollama pull {model}`. \ If the LLM runs elsewhere, set the `endpoint` input to its /api URL." )), - (Api::Openai, LlmError::Http(detail)) => ModuleError::internal(format!( - "LLM endpoint {endpoint} not reachable ({detail}). Expected an \ - OpenAI-compatible server (OpenAI, vLLM, ...) at a /v1/chat/completions URL." - )), - (Api::Anthropic, LlmError::Http(detail)) => ModuleError::internal(format!( - "LLM endpoint {endpoint} not reachable ({detail}). Expected the \ - Anthropic Messages API at a /v1/messages URL." - )), - (Api::Ollama, LlmError::Status(404)) => ModuleError::internal(format!( + LlmError::Status(404) => ModuleError::internal(format!( "LLM endpoint {endpoint} returned 404 for model '{model}' — the model is \ likely not pulled. Run `ollama pull {model}` (or check the model name)." )), - (_, LlmError::Status(401)) | (_, LlmError::Status(403)) => ModuleError::internal(format!( - "LLM endpoint {endpoint} rejected the request as unauthorized — check the \ - `api_key` input." - )), - (_, LlmError::Status(code)) => ModuleError::internal(format!( + LlmError::Status(code) => ModuleError::internal(format!( "LLM endpoint {endpoint} returned HTTP {code} for model '{model}'." )), - (_, LlmError::Decode(detail)) => ModuleError::internal(format!( - "LLM response from {endpoint} did not match the expected schema: {detail}" + LlmError::Decode(detail) => ModuleError::internal(format!( + "LLM response from {endpoint} was not valid Ollama JSON: {detail}" )), - (_, LlmError::MissingInput(name)) => { - ModuleError::invalid_input(format!("missing required input '{name}'")) - } - (_, LlmError::UnsupportedApi(raw)) => ModuleError::invalid_input(format!( - "unsupported api '{raw}' (expected: ollama, openai, anthropic)" + LlmError::MissingInput(name) => ModuleError::invalid_input(format!( + "missing required input '{name}'" )), } } @@ -130,7 +100,7 @@ impl crate::llm::LlmClient for HostStubClient { &self, _url: &str, _body: &str, - _headers: &[(&'static str, String)], + _api_key: &str, ) -> Result { Err(crate::llm::LlmError::Http( "LLM HTTP path is unavailable on the host build; only wasm32 supports outbound HTTP" @@ -148,14 +118,14 @@ impl crate::llm::LlmClient for WakiClient { &self, url: &str, body: &str, - headers: &[(&'static str, String)], + api_key: &str, ) -> Result { let mut request = waki::Client::new() .post(url) .header("Content-Type", "application/json") .body(body.to_string()); - for (name, value) in headers { - request = request.header(*name, value); + if !api_key.is_empty() { + request = request.header("Authorization", &format!("Bearer {api_key}")); } let response = request .send() diff --git a/src/llm.rs b/src/llm.rs index 90565f5..e09707f 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -1,84 +1,35 @@ -//! Multi-API chat client: Ollama (default), OpenAI, Anthropic. +//! Ollama-shaped chat client. //! -//! The wire format is selected via the optional `api` input: -//! -//! - `ollama` (default) — Ollama `/api/chat`; unchanged v0.1.0 -//! behavior including the best-effort model-digest probe. -//! - `openai` — OpenAI Chat Completions wire format -//! (`/v1/chat/completions`), the de-facto standard implemented -//! by vLLM and most self-hosted inference servers. Bearer auth. -//! - `anthropic` — Anthropic Messages API (`/v1/messages`), -//! `x-api-key` + `anthropic-version` headers, top-level -//! `system` field, mandatory `max_tokens`. -//! -//! No streaming, no tool calls — one prompt in, one completion out. +//! v0.1.0 targets the Ollama `/api/chat` endpoint. OpenAI- and +//! Anthropic-compatible adapters are deliberately deferred — each +//! has its own request/response shape that warrants its own crate +//! (or at least its own module here) once a flow needs it. //! //! All HTTP I/O lives behind a `LlmClient` trait so unit tests can -//! exercise request building, response parsing, and the digest +//! exercise prompt building, response parsing, and the digest //! probe on the host without making real network calls. use serde::Serialize; -/// Anthropic's Messages API requires `max_tokens`. This default is -/// large enough for document-processing completions while staying -/// well below every current model's output cap. -const ANTHROPIC_DEFAULT_MAX_TOKENS: u32 = 4096; - -/// Pinned Messages API version header. Anthropic keeps old -/// versions working; bump deliberately, never implicitly. -const ANTHROPIC_VERSION: &str = "2023-06-01"; - #[allow(dead_code)] #[derive(Debug, thiserror::Error)] pub enum LlmError { #[error("missing required input '{0}'")] MissingInput(&'static str), - #[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")] - UnsupportedApi(String), #[error("http error: {0}")] Http(String), #[error("non-success status: {0}")] Status(u16), - #[error("response body could not be parsed as the expected schema: {0}")] + #[error("response body could not be parsed as Ollama schema: {0}")] Decode(String), } -/// Which wire format to speak. Selected by the `api` input. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Api { - Ollama, - Openai, - Anthropic, -} - -impl Api { - /// Parse the `api` input. Empty string means the input was not - /// provided and falls back to Ollama (v0.1.0 behavior). - pub fn parse(raw: &str) -> Result { - match raw.trim().to_ascii_lowercase().as_str() { - "" | "ollama" => Ok(Api::Ollama), - "openai" => Ok(Api::Openai), - "anthropic" => Ok(Api::Anthropic), - other => Err(LlmError::UnsupportedApi(other.to_string())), - } - } -} - pub trait LlmClient { - /// POST `body` as JSON to `url` with the given extra headers. - /// `Content-Type: application/json` is implied. Header VALUES - /// may carry credentials — implementations must never log them. - fn post_json( - &self, - url: &str, - body: &str, - headers: &[(&'static str, String)], - ) -> Result; + fn post_json(&self, url: &str, body: &str, api_key: &str) -> Result; } #[derive(Debug, Clone)] pub struct ChatParams<'a> { - pub api: Api, pub endpoint: &'a str, pub model: &'a str, pub api_key: &'a str, @@ -86,29 +37,8 @@ pub struct ChatParams<'a> { pub prompt: &'a str, } -/// Auth/protocol headers for the selected API. The api_key is only -/// ever placed into a header value here; it must not appear in any -/// error, output, or log line. -pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> { - let mut headers = Vec::with_capacity(2); - match p.api { - Api::Ollama | Api::Openai => { - if !p.api_key.is_empty() { - headers.push(("Authorization", format!("Bearer {}", p.api_key))); - } - } - Api::Anthropic => { - if !p.api_key.is_empty() { - headers.push(("x-api-key", p.api_key.to_string())); - } - headers.push(("anthropic-version", ANTHROPIC_VERSION.to_string())); - } - } - headers -} - #[derive(Serialize)] -struct ChatMessage<'a> { +struct OllamaMessage<'a> { role: &'a str, content: &'a str, } @@ -116,86 +46,33 @@ struct ChatMessage<'a> { #[derive(Serialize)] struct OllamaRequest<'a> { model: &'a str, - messages: Vec>, + messages: Vec>, stream: bool, } -#[derive(Serialize)] -struct OpenAiRequest<'a> { - model: &'a str, - messages: Vec>, - stream: bool, -} - -#[derive(Serialize)] -struct AnthropicRequest<'a> { - model: &'a str, - max_tokens: u32, - #[serde(skip_serializing_if = "Option::is_none")] - system: Option<&'a str>, - messages: Vec>, -} - -/// System-then-user message list shared by the Ollama and OpenAI -/// wire formats. The system message is omitted when empty so a -/// deployment can use the model's built-in system prompt. -fn build_messages<'a>(p: &ChatParams<'a>) -> Vec> { +/// Build the request body for an Ollama `/api/chat` invocation. +/// The system prompt is omitted from the messages list when empty +/// so a deployment can use the model's built-in system prompt. +pub fn build_ollama_body(p: &ChatParams) -> String { let mut messages = Vec::with_capacity(2); if !p.system_prompt.is_empty() { - messages.push(ChatMessage { + messages.push(OllamaMessage { role: "system", content: p.system_prompt, }); } - messages.push(ChatMessage { + messages.push(OllamaMessage { role: "user", content: p.prompt, }); - messages -} - -/// Build the request body for an Ollama `/api/chat` invocation. -pub fn build_ollama_body(p: &ChatParams) -> String { let req = OllamaRequest { model: p.model, - messages: build_messages(p), + messages, stream: false, }; serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) } -/// Build the request body for an OpenAI-compatible -/// `/v1/chat/completions` invocation (OpenAI, vLLM, most -/// self-hosted inference servers). -pub fn build_openai_body(p: &ChatParams) -> String { - let req = OpenAiRequest { - model: p.model, - messages: build_messages(p), - stream: false, - }; - serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) -} - -/// Build the request body for an Anthropic `/v1/messages` -/// invocation. `system` is a top-level field (not a message); -/// `max_tokens` is mandatory in the Messages API. -pub fn build_anthropic_body(p: &ChatParams) -> String { - let req = AnthropicRequest { - model: p.model, - max_tokens: ANTHROPIC_DEFAULT_MAX_TOKENS, - system: if p.system_prompt.is_empty() { - None - } else { - Some(p.system_prompt) - }, - messages: vec![ChatMessage { - role: "user", - content: p.prompt, - }], - }; - serde_json::to_string(&req).unwrap_or_else(|_| String::from("{}")) -} - /// Extract the assistant's message text from an Ollama /// /api/chat response. pub fn extract_ollama_content(body: &str) -> Result { @@ -208,44 +85,16 @@ pub fn extract_ollama_content(body: &str) -> Result { .ok_or_else(|| LlmError::Decode("missing message.content".into())) } -/// Extract the assistant text from an OpenAI Chat Completions -/// response: `choices[0].message.content`. -pub fn extract_openai_content(body: &str) -> Result { - let v: serde_json::Value = - serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; - v.get("choices") - .and_then(|c| c.get(0)) - .and_then(|c| c.get("message")) - .and_then(|m| m.get("content")) - .and_then(|c| c.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| LlmError::Decode("missing choices[0].message.content".into())) -} - -/// Extract the assistant text from an Anthropic Messages -/// response: `content[0].text`. -pub fn extract_anthropic_content(body: &str) -> Result { - let v: serde_json::Value = - serde_json::from_str(body).map_err(|e| LlmError::Decode(e.to_string()))?; - v.get("content") - .and_then(|c| c.get(0)) - .and_then(|c| c.get("text")) - .and_then(|t| t.as_str()) - .map(|s| s.to_string()) - .ok_or_else(|| LlmError::Decode("missing content[0].text".into())) -} - #[derive(Debug, Clone)] pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, } -/// Run one chat call in the selected wire format AND, for Ollama, -/// probe the model digest. The probe is best-effort — non-Ollama -/// APIs expose no digest endpoint, so `model_digest = None` there -/// (documented empty, never invented); transient probe failures on -/// Ollama also yield `None` rather than failing the whole call. +/// Run an Ollama chat call AND probe the model digest. Probe is +/// best-effort — non-Ollama endpoints (no `/api/chat` suffix) and +/// transient failures yield `model_digest = None` rather than +/// failing the whole call. pub fn chat_with_identity( client: &C, p: &ChatParams, @@ -259,22 +108,10 @@ pub fn chat_with_identity( if p.prompt.is_empty() { return Err(LlmError::MissingInput("prompt")); } - let body = match p.api { - Api::Ollama => build_ollama_body(p), - Api::Openai => build_openai_body(p), - Api::Anthropic => build_anthropic_body(p), - }; - let headers = build_headers(p); - let response_body = client.post_json(p.endpoint, &body, &headers)?; - let response = match p.api { - Api::Ollama => extract_ollama_content(&response_body)?, - Api::Openai => extract_openai_content(&response_body)?, - Api::Anthropic => extract_anthropic_content(&response_body)?, - }; - let model_digest = match p.api { - Api::Ollama => probe_model_digest(client, p), - Api::Openai | Api::Anthropic => None, - }; + let body = build_ollama_body(p); + let response_body = client.post_json(p.endpoint, &body, p.api_key)?; + let response = extract_ollama_content(&response_body)?; + let model_digest = probe_model_digest(client, p); Ok(ChatWithIdentity { response, model_digest, @@ -284,8 +121,7 @@ pub fn chat_with_identity( fn probe_model_digest(client: &C, p: &ChatParams) -> Option { let show_url = derive_show_url(p.endpoint)?; let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?; - let headers = build_headers(p); - let response = client.post_json(&show_url, &body, &headers).ok()?; + let response = client.post_json(&show_url, &body, p.api_key).ok()?; extract_show_digest(&response) } @@ -324,39 +160,20 @@ mod tests { use super::*; use std::cell::RefCell; - /// (url, body, headers) of one recorded call. - type RecordedCall = (String, String, Vec<(String, String)>); - struct MockClient { responses: RefCell>>, - /// Every call, in call order. - calls: RefCell>, } impl MockClient { fn new(responses: Vec>) -> Self { Self { responses: RefCell::new(responses), - calls: RefCell::new(Vec::new()), } } } impl LlmClient for MockClient { - fn post_json( - &self, - url: &str, - body: &str, - headers: &[(&'static str, String)], - ) -> Result { - self.calls.borrow_mut().push(( - url.to_string(), - body.to_string(), - headers - .iter() - .map(|(k, v)| (k.to_string(), v.clone())) - .collect(), - )); + fn post_json(&self, _url: &str, _body: &str, _api_key: &str) -> Result { self.responses .borrow_mut() .pop() @@ -364,42 +181,14 @@ mod tests { } } - fn params<'a>(api: Api) -> ChatParams<'a> { - ChatParams { - api, - endpoint: "http://x/api/chat", - model: "qwen", - api_key: "", - system_prompt: "", - prompt: "hello", - } - } - - // ---------------- api input parsing ---------------- - - #[test] - fn api_parse_defaults_to_ollama() { - assert_eq!(Api::parse("").unwrap(), Api::Ollama); - assert_eq!(Api::parse("ollama").unwrap(), Api::Ollama); - assert_eq!(Api::parse(" OpenAI ").unwrap(), Api::Openai); - assert_eq!(Api::parse("anthropic").unwrap(), Api::Anthropic); - } - - #[test] - fn api_parse_rejects_unknown() { - assert!(matches!( - Api::parse("gemini"), - Err(LlmError::UnsupportedApi(_)) - )); - } - - // ---------------- Ollama wire format (regression) ---------------- - #[test] fn ollama_body_includes_system_when_provided() { let p = ChatParams { + endpoint: "http://x/api/chat", + model: "qwen", + api_key: "", system_prompt: "be helpful", - ..params(Api::Ollama) + prompt: "hello", }; let body = build_ollama_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); @@ -409,12 +198,18 @@ mod tests { assert_eq!(messages[0]["content"], "be helpful"); assert_eq!(messages[1]["role"], "user"); assert_eq!(messages[1]["content"], "hello"); - assert_eq!(v["stream"], false); } #[test] fn ollama_body_omits_system_when_empty() { - let body = build_ollama_body(¶ms(Api::Ollama)); + let p = ChatParams { + endpoint: "http://x/api/chat", + model: "qwen", + api_key: "", + system_prompt: "", + prompt: "hello", + }; + let body = build_ollama_body(&p); let v: serde_json::Value = serde_json::from_str(&body).unwrap(); let messages = v["messages"].as_array().unwrap(); assert_eq!(messages.len(), 1); @@ -436,208 +231,6 @@ mod tests { )); } - #[test] - fn ollama_bearer_header_only_with_key() { - assert!(build_headers(¶ms(Api::Ollama)).is_empty()); - let p = ChatParams { - api_key: "sk-test", - ..params(Api::Ollama) - }; - assert_eq!( - build_headers(&p), - vec![("Authorization", "Bearer sk-test".to_string())] - ); - } - - // ---------------- OpenAI wire format ---------------- - - #[test] - fn openai_body_matches_chat_completions_format() { - let p = ChatParams { - api: Api::Openai, - endpoint: "http://vllm:8000/v1/chat/completions", - model: "meta-llama/Llama-3.1-8B-Instruct", - api_key: "sk-x", - system_prompt: "be terse", - prompt: "hello", - }; - let body = build_openai_body(&p); - let v: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(v["model"], "meta-llama/Llama-3.1-8B-Instruct"); - assert_eq!(v["stream"], false); - let messages = v["messages"].as_array().unwrap(); - assert_eq!(messages.len(), 2); - assert_eq!(messages[0]["role"], "system"); - assert_eq!(messages[0]["content"], "be terse"); - assert_eq!(messages[1]["role"], "user"); - assert_eq!(messages[1]["content"], "hello"); - // No Anthropic-only fields leak into the OpenAI body. - assert!(v.get("max_tokens").is_none()); - assert!(v.get("system").is_none()); - } - - #[test] - fn openai_response_parses_fixture() { - // Shape as returned by OpenAI / vLLM /v1/chat/completions. - let body = r#"{ - "id": "chatcmpl-123", - "object": "chat.completion", - "created": 1719000000, - "model": "meta-llama/Llama-3.1-8B-Instruct", - "choices": [{ - "index": 0, - "message": {"role": "assistant", "content": "Hi from vLLM"}, - "finish_reason": "stop" - }], - "usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13} - }"#; - assert_eq!(extract_openai_content(body).unwrap(), "Hi from vLLM"); - } - - #[test] - fn openai_response_errors_on_missing_choices() { - assert!(matches!( - extract_openai_content(r#"{"object":"chat.completion","choices":[]}"#), - Err(LlmError::Decode(_)) - )); - assert!(matches!( - extract_openai_content(r#"{"error":{"message":"invalid key"}}"#), - Err(LlmError::Decode(_)) - )); - } - - #[test] - fn openai_uses_bearer_auth() { - let p = ChatParams { - api_key: "sk-test", - ..params(Api::Openai) - }; - assert_eq!( - build_headers(&p), - vec![("Authorization", "Bearer sk-test".to_string())] - ); - } - - #[test] - fn openai_chat_returns_content_and_no_digest() { - let canned = r#"{"choices":[{"message":{"role":"assistant","content":"ok"}}]}"#; - let client = MockClient::new(vec![Ok(canned.to_string())]); - let p = ChatParams { - api: Api::Openai, - endpoint: "http://vllm:8000/v1/chat/completions", - model: "m", - api_key: "sk", - system_prompt: "", - prompt: "hi", - }; - let result = chat_with_identity(&client, &p).unwrap(); - assert_eq!(result.response, "ok"); - // No digest API on OpenAI-compatible endpoints — documented - // empty, and exactly one HTTP call (no probe). - assert_eq!(result.model_digest, None); - assert_eq!(client.calls.borrow().len(), 1); - } - - // ---------------- Anthropic wire format ---------------- - - #[test] - fn anthropic_body_matches_messages_format() { - let p = ChatParams { - api: Api::Anthropic, - endpoint: "https://api.anthropic.com/v1/messages", - model: "claude-fable-5", - api_key: "sk-ant", - system_prompt: "be terse", - prompt: "hello", - }; - let body = build_anthropic_body(&p); - let v: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert_eq!(v["model"], "claude-fable-5"); - // max_tokens is mandatory in the Messages API. - assert_eq!(v["max_tokens"], 4096); - // system is a top-level field, never a message. - assert_eq!(v["system"], "be terse"); - let messages = v["messages"].as_array().unwrap(); - assert_eq!(messages.len(), 1); - assert_eq!(messages[0]["role"], "user"); - assert_eq!(messages[0]["content"], "hello"); - // No OpenAI/Ollama-only fields leak in. - assert!(v.get("stream").is_none()); - } - - #[test] - fn anthropic_body_omits_system_when_empty() { - let body = build_anthropic_body(¶ms(Api::Anthropic)); - let v: serde_json::Value = serde_json::from_str(&body).unwrap(); - assert!(v.get("system").is_none()); - assert_eq!(v["max_tokens"], 4096); - } - - #[test] - fn anthropic_response_parses_fixture() { - // Shape as returned by the Anthropic Messages API. - let body = r#"{ - "id": "msg_01XFDUDYJgAACzvnptvVoYEL", - "type": "message", - "role": "assistant", - "model": "claude-fable-5", - "content": [{"type": "text", "text": "Hi from Claude"}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 10, "output_tokens": 5} - }"#; - assert_eq!(extract_anthropic_content(body).unwrap(), "Hi from Claude"); - } - - #[test] - fn anthropic_response_errors_on_missing_content() { - assert!(matches!( - extract_anthropic_content(r#"{"type":"message","content":[]}"#), - Err(LlmError::Decode(_)) - )); - assert!(matches!( - extract_anthropic_content(r#"{"type":"error","error":{"message":"x"}}"#), - Err(LlmError::Decode(_)) - )); - } - - #[test] - fn anthropic_uses_x_api_key_and_version_headers() { - let p = ChatParams { - api_key: "sk-ant-test", - ..params(Api::Anthropic) - }; - let headers = build_headers(&p); - assert_eq!( - headers, - vec![ - ("x-api-key", "sk-ant-test".to_string()), - ("anthropic-version", "2023-06-01".to_string()), - ] - ); - // Never Bearer auth on the Anthropic path. - assert!(headers.iter().all(|(k, _)| *k != "Authorization")); - } - - #[test] - fn anthropic_chat_returns_content_and_no_digest() { - let canned = r#"{"content":[{"type":"text","text":"ok"}]}"#; - let client = MockClient::new(vec![Ok(canned.to_string())]); - let p = ChatParams { - api: Api::Anthropic, - endpoint: "https://api.anthropic.com/v1/messages", - model: "claude-fable-5", - api_key: "sk-ant", - system_prompt: "", - prompt: "hi", - }; - let result = chat_with_identity(&client, &p).unwrap(); - assert_eq!(result.response, "ok"); - assert_eq!(result.model_digest, None); - assert_eq!(client.calls.borrow().len(), 1); - } - - // ---------------- digest probe (Ollama only) ---------------- - #[test] fn derive_show_url_swaps_chat_suffix() { assert_eq!( @@ -670,7 +263,10 @@ mod tests { let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", - ..params(Api::Ollama) + model: "qwen", + api_key: "", + system_prompt: "", + prompt: "hi", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "the response"); @@ -683,7 +279,10 @@ mod tests { let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]); let p = ChatParams { endpoint: "http://localhost:11434/api/chat", - ..params(Api::Ollama) + model: "qwen", + api_key: "", + system_prompt: "", + prompt: "hi", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); @@ -691,26 +290,30 @@ mod tests { } #[test] - fn chat_with_identity_skips_probe_for_non_ollama_url() { + fn chat_with_identity_skips_probe_for_non_ollama() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string(); let client = MockClient::new(vec![Ok(canned_chat)]); let p = ChatParams { - endpoint: "https://example.com/proxy/chat", - ..params(Api::Ollama) + endpoint: "https://api.openai.com/v1/chat/completions", + model: "gpt", + api_key: "k", + system_prompt: "", + prompt: "hi", }; let result = chat_with_identity(&client, &p).unwrap(); assert_eq!(result.response, "ok"); assert_eq!(result.model_digest, None); } - // ---------------- input guards ---------------- - #[test] fn empty_prompt_is_rejected() { let client = MockClient::new(vec![]); let p = ChatParams { + endpoint: "http://x/api/chat", + model: "x", + api_key: "", + system_prompt: "", prompt: "", - ..params(Api::Ollama) }; assert!(matches!( chat_with_identity(&client, &p),