diff --git a/Cargo.toml b/Cargo.toml index 689f150..6f0614c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "orchestrator_llm" -version = "0.3.0" +version = "0.3.1" edition = "2024" authors = ["Dr. Stefan Flemming "] license = "Apache-2.0" diff --git a/module.yaml b/module.yaml index 8f78c92..f1fff80 100644 --- a/module.yaml +++ b/module.yaml @@ -1,11 +1,11 @@ schema_version: 1 name: orchestrator-llm -version: 0.2.0 +version: 0.3.1 # Capability provided by this module. provides: - capability: orchestrator.plan - version: 0.2.0 + version: 0.3.0 # Inputs the invoke function accepts. inputs: @@ -31,6 +31,17 @@ inputs: outputs: # JSON-encoded Plan compatible with `fai_hub::plan::Plan`. plan: json + # The endpoint the plan was generated against. Empty when the + # deterministic stub path (no llm_endpoint) ran. + model_endpoint: text + # The model name as supplied to the LLM API. + model_name: text + # SHA-256 digest of the served Ollama model, when reachable. + # Empty for cloud providers (OpenAI / Anthropic do not expose + # a digest API) and for the deterministic stub path. Together + # with model_endpoint and model_name this answers the audit + # question "which exact model produced this plan?" + model_digest: text # Permissions required. # diff --git a/src/lib.rs b/src/lib.rs index 7770e0d..900660c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,8 +45,8 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { .and_then(payload_text) .unwrap_or_default(); - let plan_json = if llm_endpoint.is_empty() { - build_stub_plan(goal) + let (plan_json, model_digest) = if llm_endpoint.is_empty() { + (build_stub_plan(goal), None) } else { let client = make_client(); let params = crate::llm::OllamaParams { @@ -57,14 +57,16 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { available_capabilities: &available_capabilities, store_capabilities: &store_capabilities, }; - crate::llm::generate_plan(&client, ¶ms) - .map_err(|e| ModuleError::internal(e.to_string()))? + let result = crate::llm::generate_plan_with_identity(&client, ¶ms) + .map_err(|e| ModuleError::internal(e.to_string()))?; + (result.plan_json, result.model_digest) }; Ok(Outputs::new() .with_json_str("plan", plan_json) .with_text("model_endpoint", llm_endpoint) - .with_text("model_name", llm_model)) + .with_text("model_name", llm_model) + .with_text("model_digest", model_digest.unwrap_or_default())) } fn payload_text(p: &Payload) -> Option { diff --git a/src/llm.rs b/src/llm.rs index df6fd88..55a5f58 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -127,12 +127,43 @@ pub fn extract_ollama_content(body: &str) -> Result { .ok_or_else(|| LlmError::Decode("missing message.content".into())) } +/// Plan plus model identity captured at invocation time, for +/// audit logging. +#[derive(Debug, Clone)] +pub struct PlanWithIdentity { + /// The plan JSON the LLM produced. + pub plan_json: String, + /// SHA-256 digest of the served model, when reachable. `None` + /// for non-Ollama endpoints (OpenAI / Anthropic do not expose + /// a per-model digest API) or when the `/api/show` probe failed + /// for any reason. The audit trail records this verbatim; + /// `None` becomes an empty string in the module output. + pub model_digest: Option, +} + /// Generate a plan via the configured LLM endpoint. Returns the /// validated plan JSON ready to be emitted as the module output. +/// +/// Convenience wrapper kept for callers that do not care about +/// the model digest. New code should call `generate_plan_with_identity`. +#[allow(dead_code)] pub fn generate_plan( client: &C, p: &OllamaParams, ) -> Result { + generate_plan_with_identity(client, p).map(|r| r.plan_json) +} + +/// Generate a plan AND probe the LLM endpoint for the model's +/// SHA-256 digest. The digest probe is best-effort: failures do +/// not propagate, the digest is simply absent from the result. +/// This gives compliance-grade audit on Ollama deployments while +/// staying compatible with cloud providers that do not expose +/// digests. +pub fn generate_plan_with_identity( + client: &C, + p: &OllamaParams, +) -> Result { if p.endpoint.is_empty() { return Err(OrchestratorError::MissingEndpoint); } @@ -146,7 +177,59 @@ pub fn generate_plan( .map_err(OrchestratorError::Llm)?; let plan_json = extract_ollama_content(&response_body).map_err(OrchestratorError::Llm)?; crate::plan::parse_and_validate(&plan_json).map_err(OrchestratorError::Plan)?; - Ok(plan_json) + + let model_digest = probe_model_digest(client, p); + Ok(PlanWithIdentity { + plan_json, + model_digest, + }) +} + +/// Best-effort probe of Ollama's `/api/show` for the model digest. +/// Any failure (non-Ollama endpoint, network error, parse failure) +/// returns `None` — never propagates as an error to the caller. +fn probe_model_digest(client: &C, p: &OllamaParams) -> Option { + let show_url = derive_show_url(p.endpoint)?; + let body = serde_json::to_string(&serde_json::json!({ "name": p.model })).ok()?; + let response = client.post_json(&show_url, &body, p.api_key).ok()?; + extract_show_digest(&response) +} + +/// Convert an Ollama `/api/chat` URL into the matching +/// `/api/show` URL by suffix substitution. Returns `None` if the +/// endpoint does not end in `/api/chat` — that is the signal that +/// the endpoint is not Ollama-shaped, so a probe makes no sense. +pub fn derive_show_url(chat_endpoint: &str) -> Option { + if chat_endpoint.ends_with("/api/chat") { + let head_len = chat_endpoint.len() - "/api/chat".len(); + let mut url = String::with_capacity(head_len + "/api/show".len()); + url.push_str(&chat_endpoint[..head_len]); + url.push_str("/api/show"); + Some(url) + } else { + None + } +} + +/// Extract the SHA-256 digest from a `/api/show` response body. +/// Looks for the field at top level, then under `details.digest`, +/// then under `model_info.digest`. Returns `None` if no candidate +/// resolves to a non-empty string. +pub fn extract_show_digest(body: &str) -> Option { + let v: serde_json::Value = serde_json::from_str(body).ok()?; + let candidates = [ + v.get("digest"), + v.get("details").and_then(|d| d.get("digest")), + v.get("model_info").and_then(|d| d.get("digest")), + ]; + for cand in candidates { + if let Some(s) = cand.and_then(|v| v.as_str()) { + if !s.is_empty() { + return Some(s.to_string()); + } + } + } + None } #[derive(Debug, thiserror::Error)] @@ -351,4 +434,123 @@ mod tests { Err(OrchestratorError::Llm(LlmError::MissingInput("llm_model"))) )); } + + // === Compliance Gap 1: model digest probe === + + #[test] + fn derive_show_url_swaps_chat_suffix() { + assert_eq!( + derive_show_url("http://localhost:11434/api/chat").as_deref(), + Some("http://localhost:11434/api/show") + ); + assert_eq!( + derive_show_url("https://example.com/v1/api/chat").as_deref(), + Some("https://example.com/v1/api/show") + ); + } + + #[test] + fn derive_show_url_returns_none_for_non_ollama_endpoints() { + assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); + assert!(derive_show_url("https://api.anthropic.com/v1/messages").is_none()); + assert!(derive_show_url("http://localhost:11434/api/chats").is_none()); + assert!(derive_show_url("/api/chat-something").is_none()); + } + + #[test] + fn extract_show_digest_finds_top_level_field() { + let body = r#"{"digest":"sha256:abcdef","details":{"format":"gguf"}}"#; + assert_eq!(extract_show_digest(body).as_deref(), Some("sha256:abcdef")); + } + + #[test] + fn extract_show_digest_falls_back_to_nested_locations() { + let nested_details = r#"{"details":{"digest":"sha256:nested-details"}}"#; + assert_eq!( + extract_show_digest(nested_details).as_deref(), + Some("sha256:nested-details") + ); + let nested_info = r#"{"model_info":{"digest":"sha256:nested-info"}}"#; + assert_eq!( + extract_show_digest(nested_info).as_deref(), + Some("sha256:nested-info") + ); + } + + #[test] + fn extract_show_digest_returns_none_when_absent_or_empty() { + assert!(extract_show_digest(r#"{"modelfile":"..."}"#).is_none()); + assert!(extract_show_digest(r#"{"digest":""}"#).is_none()); + assert!(extract_show_digest("not even json").is_none()); + } + + #[test] + fn generate_plan_with_identity_records_digest_when_show_responds() { + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}},"done":true}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); + // Mock pops from the end — so push show first, chat second. + let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "http://localhost:11434/api/chat", + model: "qwen", + api_key: "", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); + assert!(!result.plan_json.is_empty()); + } + + #[test] + fn generate_plan_with_identity_returns_none_digest_for_non_ollama_endpoint() { + // Endpoint does not end in /api/chat: probe is skipped + // entirely. Only one mock response is consumed. + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}}}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let client = MockClient::new(vec![Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "https://api.openai.com/v1/chat/completions", + model: "gpt", + api_key: "k", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest, None); + } + + #[test] + fn generate_plan_with_identity_swallows_show_failures() { + // The /api/show probe fails (e.g. 500). The plan still + // succeeds with `model_digest = None`. + let canned_plan = + r#"{"schema_version":1,"goal":"x","steps":[{"kind":"explain","text":"ok"}]}"#; + let canned_chat = format!( + r#"{{"message":{{"content":{}}}}}"#, + serde_json::to_string(canned_plan).unwrap(), + ); + let client = MockClient::new(vec![Err(LlmError::Status(500)), Ok(canned_chat)]); + let p = OllamaParams { + endpoint: "http://localhost:11434/api/chat", + model: "qwen", + api_key: "", + goal: "x", + available_capabilities: "", + store_capabilities: "", + }; + let result = generate_plan_with_identity(&client, &p).unwrap(); + assert_eq!(result.model_digest, None); + } }