From 86eb72154e11df196c8c16a07fb5a108236fccc6 Mon Sep 17 00:00:00 2001 From: flemming-it Date: Sun, 3 May 2026 23:06:27 +0200 Subject: [PATCH] feat: probe Ollama model digest, emit as model_digest output (v0.3.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes Compliance Gap 1 from fai/platform :: docs/advanced/ compliance-gaps.md. The module now probes the LLM endpoint's /api/show on every successful chat invocation and surfaces the model's SHA-256 digest as a separate output field. The probe is strictly best-effort: - derive_show_url returns None unless the configured endpoint ends in /api/chat, so non-Ollama deployments (OpenAI, Anthropic) skip the probe entirely. - probe_model_digest swallows every failure path (network error, non-200 status, malformed JSON, missing digest field) and yields None. The plan generation succeeds either way. - extract_show_digest looks for digest at top level, then details.digest, then model_info.digest — the fields shift across Ollama versions, so the lookup is tolerant. A new generate_plan_with_identity returns a PlanWithIdentity struct ({plan_json, model_digest}). The original generate_plan remains as a #[allow(dead_code)] convenience wrapper that returns just the plan JSON. The invoke entry point in lib.rs now emits four outputs: plan (json) — unchanged model_endpoint (text) — unchanged model_name (text) — unchanged model_digest (text) — NEW; empty for non-Ollama or stub paths module.yaml documents the new output and bumps the orchestrator capability version to 0.3.0 (additive minor; downstream flows that referenced @^0 keep working). Eight new tests in src/llm.rs cover the new code paths: - URL transform (/api/chat → /api/show) - URL transform skipped for non-Ollama endpoints - Digest extraction at all three documented JSON locations - Digest extraction graceful failure (missing, empty, bad JSON) - End-to-end probe success - End-to-end probe skipped for non-Ollama - End-to-end probe failure swallowed without breaking plan Total tests: 26 (was 18). Wasm artifact builds with v1.0 imports baked in. Platform-side integration tests in fai/platform :: orchestrator_llm_path.rs pass against the new module unchanged. Bumps the module to 0.3.1 (patch — output schema additively extended, behaviour unchanged for existing flows). Signed-off-by: flemming-it --- Cargo.toml | 2 +- module.yaml | 15 +++- src/lib.rs | 12 ++-- src/llm.rs | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 224 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 689f150..6f0614c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "orchestrator_llm" -version = "0.3.0" +version = "0.3.1" edition = "2024" authors = ["Dr. Stefan Flemming "] license = "Apache-2.0" diff --git a/module.yaml b/module.yaml index 8f78c92..f1fff80 100644 --- a/module.yaml +++ b/module.yaml @@ -1,11 +1,11 @@ schema_version: 1 name: orchestrator-llm -version: 0.2.0 +version: 0.3.1 # Capability provided by this module. provides: - capability: orchestrator.plan - version: 0.2.0 + version: 0.3.0 # Inputs the invoke function accepts. inputs: @@ -31,6 +31,17 @@ inputs: outputs: # JSON-encoded Plan compatible with `fai_hub::plan::Plan`. plan: json + # The endpoint the plan was generated against. Empty when the + # deterministic stub path (no llm_endpoint) ran. + model_endpoint: text + # The model name as supplied to the LLM API. + model_name: text + # SHA-256 digest of the served Ollama model, when reachable. + # Empty for cloud providers (OpenAI / Anthropic do not expose + # a digest API) and for the deterministic stub path. Together + # with model_endpoint and model_name this answers the audit + # question "which exact model produced this plan?" + model_digest: text # Permissions required. # diff --git a/src/lib.rs b/src/lib.rs index 7770e0d..900660c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,8 +45,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .and_then(payload_text) .unwrap_or_default(); - let plan_json = if llm_endpoint.is_empty() { - build_stub_plan(goal) + let (plan_json, model_digest) = if llm_endpoint.is_empty() { + (build_stub_plan(goal), None) } else { let client = make_client(); let params = crate::llm::OllamaParams { @@ -57,14 +57,16 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { available_capabilities: &available_capabilities, store_capabilities: &store_capabilities, }; - crate::llm::generate_plan(&client, ¶ms) - .map_err(|e| ModuleError::internal(e.to_string()))? + let result = crate::llm::generate_plan_with_identity(&client, ¶ms) + .map_err(|e| ModuleError::internal(e.to_string()))?; + (result.plan_json, result.model_digest) }; Ok(Outputs::new() .with_json_str("plan", plan_json) .with_text("model_endpoint", llm_endpoint) - .with_text("model_name", llm_model)) + .with_text("model_name", llm_model) + .with_text("model_digest", model_digest.unwrap_or_default())) } fn payload_text(p: &Payload) -> Option { diff --git a/src/llm.rs b/src/llm.rs index df6fd88..55a5f58 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -127,12 +127,43 @@ pub fn extract_ollama_content(body: &str) -> Result { .ok_or_else(|| LlmError::Decode("missing message.content".into())) } +/// Plan plus model identity captured at invocation time, for +/// audit logging. +#[derive(Debug, Clone)] +pub struct PlanWithIdentity { + /// The plan JSON the LLM produced. + pub plan_json: String, + /// SHA-256 digest of the served model, when reachable. `None` + /// for non-Ollama endpoints (OpenAI / Anthropic do not expose + /// a per-model digest API) or when the `/api/show` probe failed + /// for any reason. The audit trail records this verbatim; + /// `None` becomes an empty string in the module output. + pub model_digest: Option, +} + /// Generate a plan via the configured LLM endpoint. Returns the /// validated plan JSON ready to be emitted as the module output. +/// +/// Convenience wrapper kept for callers that do not care about +/// the model digest. New code should call `generate_plan_with_identity`. +#[allow(dead_code)] pub fn generate_plan( client: &C, p: &OllamaParams, ) -> Result { + generate_plan_with_identity(client, p).map(|r| r.plan_json) +} + +/// Generate a plan AND probe the LLM endpoint for the model's +/// SHA-256 digest. The digest probe is best-effort: failures do +/// not propagate, the digest is simply absent from the result. +/// This gives compliance-grade audit on Ollama deployments while +/// staying compatible with cloud providers that do not expose +/// digests. +pub fn generate_plan_with_identity( + client: &C, + p: &OllamaParams, +) -> Result { if p.endpoint.is_empty() { return Err(OrchestratorError::MissingEndpoint); } @@ -146,7 +177,59 @@ pub fn generate_plan( .map_err(OrchestratorError::Llm)?; let plan_json = extract_ollama_content(&response_body).map_err(OrchestratorError::Llm)?; crate::plan::parse_and_validate(&plan_json).map_err(OrchestratorError::Plan)?; - Ok(plan_json) + + let model_digest = probe_model_digest(client, p); + Ok(PlanWithIdentity { + plan_json, + model_digest, + }) +} + +/// Best-effort probe of Ollama's `/api/show` for the model digest. +/// Any failure (non-Ollama endpoint, network error, parse failure) +/// returns `None` — never propagates as an error to the caller. +fn probe_model_digest(client: &C, p: &OllamaParams) -> Option { + let show_url = derive_show_url(p.endpoint)?; + let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?; + let response = client.post_json(&show_url, &body, p.api_key).ok()?; + extract_show_digest(&response) +} + +/// Convert an Ollama `/api/chat` URL into the matching +/// `/api/show` URL by suffix substitution. Returns `None` if the +/// endpoint does not end in `/api/chat` — that is the signal that +/// the endpoint is not Ollama-shaped, so a probe makes no sense. +pub fn derive_show_url(chat_endpoint: &str) -> Option { + if chat_endpoint.ends_with("/api/chat") { + let head_len = chat_endpoint.len() - "/api/chat".len(); + let mut url = String::with_capacity(head_len + "/api/show".len()); + url.push_str(&chat_endpoint[..head_len]); + url.push_str("/api/show"); + Some(url) + } else { + None + } +} + +/// Extract the SHA-256 digest from a `/api/show` response body. +/// Looks for the field at top level, then under `details.digest`, +/// then under `model_info.digest`. Returns `None` if no candidate +/// resolves to a non-empty string. +pub fn extract_show_digest(body: &str) -> Option { + let v: serde_json::Value = serde_json::from_str(body).ok()?; + let candidates = [ + v.get("digest"), + v.get("details").and_then(|d| d.get("digest")), + v.get("model_info").and_then(|d| d.get("digest")), + ]; + for cand in candidates { + if let Some(s) = cand.and_then(|v| v.as_str()) { + if !s.is_empty() { + return Some(s.to_string()); + } + } + } + None } #[derive(Debug, thiserror::Error)] @@ -351,4 +434,123 @@ mod tests { Err(OrchestratorError::Llm(LlmError::MissingInput("llm_model"))) )); } + + // === Compliance Gap 1: model digest probe === + + #[test] + fn derive_show_url_swaps_chat_suffix() { + assert_eq!( + derive_show_url("http://localhost:11434/api/chat").as_deref(), + Some("http://localhost:11434/api/show") + ); + assert_eq!( + derive_show_url("https://example.com/v1/api/chat").as_deref(), + Some("https://example.com/v1/api/show") + ); + } + + #[test] + fn derive_show_url_returns_none_for_non_ollama_endpoints() { + assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); + assert!(derive_show_url("https://api.anthropic.com/v1/messages").is_none()); + assert!(derive_show_url("http://localhost:11434/api/chats").is_none()); + assert!(derive_show_url("/api/chat-something").is_none()); + } + + #[test] + fn extract_show_digest_finds_top_level_field() { + let body = r#"{"digest":"sha256:abcdef","details":{"format":"gguf"}}"#; + assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:abcdef")); + } + + #[test] + fn extract_show_digest_falls_back_to_nested_locations() { + let nested_details = r#"{"details":{"digest":"sha256:nested-details"}}"#; + assert_eq!( + extract_show_digest(nested_details).as_deref(), + Some("sha256:nested-details") + ); + let nested_info = r#"{"model_info":{"digest":"sha256:nested-info"}}"#; + assert_eq!( + extract_show_digest(nested_info).as_deref(), + Some("sha256:nested-info") + ); + } + + #[test] + fn extract_show_digest_returns_none_when_absent_or_empty() { + assert!(extract_show_digest(r#"{"modelfile":"..."}"#).is_none()); + assert!(extract_show_digest(r#"{"digest":""}"#).is_none()); + assert!(extract_show_digest("not even json").is_none()); + } + + #[test] + fn generate_plan_with_identity_records_digest_when_show_responds() { + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}},"done":true}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); + // Mock pops from the end — so push show first, chat second. + let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "http://localhost:11434/api/chat", + model: "qwen", + api_key: "", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); + assert!(!result.plan_json.is_empty()); + } + + #[test] + fn generate_plan_with_identity_returns_none_digest_for_non_ollama_endpoint() { + // Endpoint does not end in /api/chat: probe is skipped + // entirely. Only one mock response is consumed. + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}}}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let client = MockClient::new(vec![Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "https://api.openai.com/v1/chat/completions", + model: "gpt", + api_key: "k", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest, None); + } + + #[test] + fn generate_plan_with_identity_swallows_show_failures() { + // The /api/show probe fails (e.g. 500). The plan still + // succeeds with `model_digest = None`. + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}}}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "http://localhost:11434/api/chat", + model: "qwen", + api_key: "", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest, None); + } }