diff --git a/Cargo.lock b/Cargo.lock index c510695..213beda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545" [[package]] name = "llm_chat" -version = "0.2.1" +version = "0.3.0" dependencies = [ "chain-module-sdk", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6391d14..a9ed91e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ [package] name = "llm_chat" -version = "0.2.1" +version = "0.3.0" edition = "2024" license = "Apache-2.0" publish = false diff --git a/MODULE.de.md b/MODULE.de.md index 21c2d00..8e1bb5b 100644 --- a/MODULE.de.md +++ b/MODULE.de.md @@ -10,7 +10,7 @@ die Anthropic Messages API. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.3.0` ## Eingaben @@ -40,6 +40,8 @@ die Anthropic Messages API. | `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. | | `model_name` | text | Modell-ID wie an die LLM-API gesendet. | | `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. | +| `tokens_prompt` | text | Was die Anfrage gekostet hat, nach der Zählung des Backends. Leer, wenn die Antwort keine Zählung enthielt. | +| `tokens_completion` | text | Was die Antwort gekostet hat, gleiche Regel. Ollama zählt in `eval_count`, das OpenAI-Format in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Zusammen beantworten die drei `model_*`-Ausgaben die Audit- Frage: „Welches genaue Modell hat diese Antwort erzeugt?" — diff --git a/MODULE.md b/MODULE.md index 29a511e..0d100d9 100644 --- a/MODULE.md +++ b/MODULE.md @@ -10,7 +10,7 @@ Anthropic Messages API. ## Capability -- `llm.chat@0.2.0` +- `llm.chat@0.3.0` ## Inputs @@ -40,6 +40,8 @@ Anthropic Messages API. | `model_endpoint` | text | The endpoint the response was generated against. | | `model_name` | text | The model identifier as supplied to the LLM API. | | `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. | +| `tokens_prompt` | text | What the prompt cost, as the backend counted it. Empty when the answer carried no count. | +| `tokens_completion` | text | What the answer cost, same rule. Ollama counts in `eval_count`, the OpenAI shape in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. | Together the three `model_*` outputs answer the audit question: "which exact model produced this response?" — that diff --git a/module.wasm b/module.wasm index b7e865b..d4243c2 100644 Binary files a/module.wasm and b/module.wasm differ diff --git a/module.yaml b/module.yaml index 43727f8..0a471d7 100644 --- a/module.yaml +++ b/module.yaml @@ -1,12 +1,12 @@ schema_version: 3 provider: chain name: llm-chat -version: 0.2.2 +version: 0.3.0 # Capability provided by this module. provides: - capability: llm.chat - version: 0.2.2 + version: 0.3.0 # Declared inputs in verbose v3 form. Studio renders the # description in the active locale as a tooltip on each @@ -171,6 +171,32 @@ outputs: keine Digest-API bereit). Zusammen mit model_endpoint und model_name beantwortet das die Audit-Frage "welches genaue Modell hat diese Antwort erzeugt?" + tokens_prompt: + type: text + description: + en: | + How many tokens the prompt cost, as the backend counted it. + Empty when the answer carried no count: a zero would read + like a measurement nobody took. The hub lifts this field + into the audit log automatically. + de: | + Wie viele Token die Anfrage gekostet hat, nach der Zählung + des Backends. Leer, wenn die Antwort keine Zählung enthielt: + eine Null würde wie eine Messung aussehen, die niemand + gemacht hat. Der Hub hebt dieses Feld selbst in den Prüfpfad. + tokens_completion: + type: text + description: + en: | + How many tokens the answer cost, same rules as + tokens_prompt. Ollama counts in `eval_count`, the OpenAI + shape in `usage.completion_tokens`, Anthropic in + `usage.output_tokens`. + de: | + Wie viele Token die Antwort gekostet hat, gleiche Regeln wie + tokens_prompt. Ollama zählt in `eval_count`, das + OpenAI-Format in `usage.completion_tokens`, Anthropic in + `usage.output_tokens`. # Permissions required. # diff --git a/src/lib.rs b/src/lib.rs index 4b31f50..4c3a7aa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -76,11 +76,20 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result { let result = crate::llm::chat_with_identity(&client, ¶ms) .map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?; + // The two token counts are the cost of this call, as the backend + // counted it. They stay empty where the answer carried no count, + // because an invented zero would look like a measurement. + let (tokens_prompt, tokens_completion) = match result.tokens { + Some((prompt, completion)) => (prompt.to_string(), completion.to_string()), + None => (String::new(), String::new()), + }; Ok(Outputs::new() .with_text("response", result.response) .with_text("model_endpoint", endpoint) .with_text("model_name", model) - .with_text("model_digest", result.model_digest.unwrap_or_default())) + .with_text("model_digest", result.model_digest.unwrap_or_default()) + .with_text("tokens_prompt", tokens_prompt) + .with_text("tokens_completion", tokens_completion)) } fn payload_text(p: &Payload) -> Option { diff --git a/src/llm.rs b/src/llm.rs index bb569e4..c435240 100644 --- a/src/llm.rs +++ b/src/llm.rs @@ -393,6 +393,41 @@ pub fn extract_anthropic_content(body: &str) -> Result { pub struct ChatWithIdentity { pub response: String, pub model_digest: Option, + /// How many tokens the call cost, as the backend counted them: + /// (prompt, completion). `None` where the answer carries no + /// count, which is honest about a number nobody measured. + pub tokens: Option<(u64, u64)>, +} + +/// The token counts of one answer, per wire format. +/// +/// Three dialects, three spellings of the same two numbers: Ollama +/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape +/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic +/// in `usage.input_tokens` and `usage.output_tokens`. A missing +/// count is `None` rather than a zero, because zero is a +/// measurement and this is the absence of one. +pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> { + let v: serde_json::Value = serde_json::from_str(body).ok()?; + let pair = match api { + Api::Ollama => ( + v.get("prompt_eval_count").and_then(|n| n.as_u64()), + v.get("eval_count").and_then(|n| n.as_u64()), + ), + Api::Openai | Api::Vllm => ( + v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()), + v.pointer("/usage/completion_tokens") + .and_then(|n| n.as_u64()), + ), + Api::Anthropic => ( + v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()), + v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()), + ), + }; + match pair { + (None, None) => None, + (prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))), + } } /// Run one chat call in the selected wire format AND, for Ollama, @@ -429,9 +464,11 @@ pub fn chat_with_identity( Api::Ollama => probe_model_digest(client, p), Api::Openai | Api::Vllm | Api::Anthropic => None, }; + let tokens = extract_tokens(p.api, &response_body); Ok(ChatWithIdentity { response, model_digest, + tokens, }) } @@ -815,6 +852,47 @@ mod tests { assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none()); } + /// Three dialects, three spellings of the same two numbers. + #[test] + fn every_dialect_reports_its_token_counts() { + let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#; + assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7))); + + let openai = r#"{"choices":[{"message":{"content":"ja"}}], + "usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#; + assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7))); + assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7))); + + let anthropic = r#"{"content":[{"text":"ja"}], + "usage":{"input_tokens":31,"output_tokens":7}}"#; + assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7))); + } + + /// A backend that counts nothing leaves the fields empty, rather + /// than reporting a zero nobody measured. + #[test] + fn an_answer_without_counts_reports_none() { + assert_eq!( + extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#), + None + ); + assert_eq!( + extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#), + None + ); + assert_eq!(extract_tokens(Api::Anthropic, "not json"), None); + } + + /// One count present and the other missing still says what was + /// measured. + #[test] + fn a_half_count_is_reported_as_far_as_it_goes() { + assert_eq!( + extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#), + Some((31, 0)) + ); + } + #[test] fn extract_show_digest_finds_top_level() { let body = r#"{"digest":"sha256:abc"}"#; @@ -841,6 +919,37 @@ mod tests { assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef")); } + /// The counts of the answer reach the caller, not only the + /// extractor: the cost of a call belongs in the trail. + #[test] + fn chat_with_identity_carries_the_token_counts() { + let canned_chat = r#"{"message":{"content":"ja"}, + "prompt_eval_count":31,"eval_count":7}"# + .to_string(); + let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string(); + let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]); + let p = ChatParams { + endpoint: "http://localhost:11434/api/chat", + ..params(Api::Ollama) + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.tokens, Some((31, 7))); + } + + /// A backend that counts nothing leaves the caller with nothing + /// to report, rather than with a zero. + #[test] + fn chat_with_identity_reports_no_counts_when_there_are_none() { + let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string(); + let client = MockClient::new(vec![Ok(canned_chat)]); + let p = ChatParams { + endpoint: "https://api.openai.com/v1/chat/completions", + ..params(Api::Openai) + }; + let result = chat_with_identity(&client, &p).unwrap(); + assert_eq!(result.tokens, None); + } + #[test] fn chat_with_identity_swallows_show_failure() { let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();