feat: report what a call cost in tokens

Three dialects spell the same two numbers differently: Ollama counts
in `prompt_eval_count` and `eval_count`, the OpenAI shape in
`usage.prompt_tokens` and `usage.completion_tokens`, Anthropic in
`usage.input_tokens` and `usage.output_tokens`. All three now reach
the caller as `tokens_prompt` and `tokens_completion`, which the hub
lifts into the audit log by itself.

An answer that carries no count leaves both fields empty rather than
reporting a zero: a zero is a measurement, and this is the absence of
one. Half a count is reported as far as it goes.

The cost of a model call was the one part of the audit question
"which model produced this, and what did it cost" that nothing
answered (model pool M3).

Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
flemming-it 2026-10-26 03:54:39 +01:00
parent a6fe962916
commit d5b000e18e
8 changed files with 155 additions and 7 deletions

View file

@ -76,11 +76,20 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
let result = crate::llm::chat_with_identity(&client, &params)
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
// The two token counts are the cost of this call, as the backend
// counted it. They stay empty where the answer carried no count,
// because an invented zero would look like a measurement.
let (tokens_prompt, tokens_completion) = match result.tokens {
Some((prompt, completion)) => (prompt.to_string(), completion.to_string()),
None => (String::new(), String::new()),
};
Ok(Outputs::new()
.with_text("response", result.response)
.with_text("model_endpoint", endpoint)
.with_text("model_name", model)
.with_text("model_digest", result.model_digest.unwrap_or_default()))
.with_text("model_digest", result.model_digest.unwrap_or_default())
.with_text("tokens_prompt", tokens_prompt)
.with_text("tokens_completion", tokens_completion))
}
fn payload_text(p: &Payload) -> Option<String> {

View file

@ -393,6 +393,41 @@ pub fn extract_anthropic_content(body: &str) -> Result<String, LlmError> {
pub struct ChatWithIdentity {
pub response: String,
pub model_digest: Option<String>,
/// How many tokens the call cost, as the backend counted them:
/// (prompt, completion). `None` where the answer carries no
/// count, which is honest about a number nobody measured.
pub tokens: Option<(u64, u64)>,
}
/// The token counts of one answer, per wire format.
///
/// Three dialects, three spellings of the same two numbers: Ollama
/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape
/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic
/// in `usage.input_tokens` and `usage.output_tokens`. A missing
/// count is `None` rather than a zero, because zero is a
/// measurement and this is the absence of one.
pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> {
let v: serde_json::Value = serde_json::from_str(body).ok()?;
let pair = match api {
Api::Ollama => (
v.get("prompt_eval_count").and_then(|n| n.as_u64()),
v.get("eval_count").and_then(|n| n.as_u64()),
),
Api::Openai | Api::Vllm => (
v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()),
v.pointer("/usage/completion_tokens")
.and_then(|n| n.as_u64()),
),
Api::Anthropic => (
v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()),
v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()),
),
};
match pair {
(None, None) => None,
(prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))),
}
}
/// Run one chat call in the selected wire format AND, for Ollama,
@ -429,9 +464,11 @@ pub fn chat_with_identity<C: LlmClient>(
Api::Ollama => probe_model_digest(client, p),
Api::Openai | Api::Vllm | Api::Anthropic => None,
};
let tokens = extract_tokens(p.api, &response_body);
Ok(ChatWithIdentity {
response,
model_digest,
tokens,
})
}
@ -815,6 +852,47 @@ mod tests {
assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none());
}
/// Three dialects, three spellings of the same two numbers.
#[test]
fn every_dialect_reports_its_token_counts() {
let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#;
assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7)));
let openai = r#"{"choices":[{"message":{"content":"ja"}}],
"usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#;
assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7)));
assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7)));
let anthropic = r#"{"content":[{"text":"ja"}],
"usage":{"input_tokens":31,"output_tokens":7}}"#;
assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7)));
}
/// A backend that counts nothing leaves the fields empty, rather
/// than reporting a zero nobody measured.
#[test]
fn an_answer_without_counts_reports_none() {
assert_eq!(
extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#),
None
);
assert_eq!(
extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#),
None
);
assert_eq!(extract_tokens(Api::Anthropic, "not json"), None);
}
/// One count present and the other missing still says what was
/// measured.
#[test]
fn a_half_count_is_reported_as_far_as_it_goes() {
assert_eq!(
extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#),
Some((31, 0))
);
}
#[test]
fn extract_show_digest_finds_top_level() {
let body = r#"{"digest":"sha256:abc"}"#;
@ -841,6 +919,37 @@ mod tests {
assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef"));
}
/// The counts of the answer reach the caller, not only the
/// extractor: the cost of a call belongs in the trail.
#[test]
fn chat_with_identity_carries_the_token_counts() {
let canned_chat = r#"{"message":{"content":"ja"},
"prompt_eval_count":31,"eval_count":7}"#
.to_string();
let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string();
let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]);
let p = ChatParams {
endpoint: "http://localhost:11434/api/chat",
..params(Api::Ollama)
};
let result = chat_with_identity(&client, &p).unwrap();
assert_eq!(result.tokens, Some((31, 7)));
}
/// A backend that counts nothing leaves the caller with nothing
/// to report, rather than with a zero.
#[test]
fn chat_with_identity_reports_no_counts_when_there_are_none() {
let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string();
let client = MockClient::new(vec![Ok(canned_chat)]);
let p = ChatParams {
endpoint: "https://api.openai.com/v1/chat/completions",
..params(Api::Openai)
};
let result = chat_with_identity(&client, &p).unwrap();
assert_eq!(result.tokens, None);
}
#[test]
fn chat_with_identity_swallows_show_failure() {
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();