feat: report what a call cost in tokens
Three dialects spell the same two numbers differently: Ollama counts in `prompt_eval_count` and `eval_count`, the OpenAI shape in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic in `usage.input_tokens` and `usage.output_tokens`. All three now reach the caller as `tokens_prompt` and `tokens_completion`, which the hub lifts into the audit log by itself. An answer that carries no count leaves both fields empty rather than reporting a zero: a zero is a measurement, and this is the absence of one. Half a count is reported as far as it goes. The cost of a model call was the one part of the audit question "which model produced this, and what did it cost" that nothing answered (model pool M3). Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
a6fe962916
commit
d5b000e18e
8 changed files with 155 additions and 7 deletions
2
Cargo.lock
generated
2
Cargo.lock
generated
|
|
@ -138,7 +138,7 @@ checksum = "6cc46bac87ef8093eed6f272babb833b6443374399985ac8ed28471ee0918545"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "llm_chat"
|
name = "llm_chat"
|
||||||
version = "0.2.1"
|
version = "0.3.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chain-module-sdk",
|
"chain-module-sdk",
|
||||||
"serde",
|
"serde",
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@
|
||||||
|
|
||||||
[package]
|
[package]
|
||||||
name = "llm_chat"
|
name = "llm_chat"
|
||||||
version = "0.2.1"
|
version = "0.3.0"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
publish = false
|
publish = false
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ die Anthropic Messages API.
|
||||||
|
|
||||||
## Capability
|
## Capability
|
||||||
|
|
||||||
- `llm.chat@0.2.0`
|
- `llm.chat@0.3.0`
|
||||||
|
|
||||||
## Eingaben
|
## Eingaben
|
||||||
|
|
||||||
|
|
@ -40,6 +40,8 @@ die Anthropic Messages API.
|
||||||
| `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. |
|
| `model_endpoint` | text | URL, gegen die die Antwort erzeugt wurde. |
|
||||||
| `model_name` | text | Modell-ID wie an die LLM-API gesendet. |
|
| `model_name` | text | Modell-ID wie an die LLM-API gesendet. |
|
||||||
| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. |
|
| `model_digest` | text | SHA-256-Digest des bedienenden Ollama-Modells. Leer bei `openai`/`anthropic` — diese APIs bieten keinen Digest; das Feld bleibt leer statt erfunden. |
|
||||||
|
| `tokens_prompt` | text | Was die Anfrage gekostet hat, nach der Zählung des Backends. Leer, wenn die Antwort keine Zählung enthielt. |
|
||||||
|
| `tokens_completion` | text | Was die Antwort gekostet hat, gleiche Regel. Ollama zählt in `eval_count`, das OpenAI-Format in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. |
|
||||||
|
|
||||||
Zusammen beantworten die drei `model_*`-Ausgaben die Audit-
|
Zusammen beantworten die drei `model_*`-Ausgaben die Audit-
|
||||||
Frage: „Welches genaue Modell hat diese Antwort erzeugt?" —
|
Frage: „Welches genaue Modell hat diese Antwort erzeugt?" —
|
||||||
|
|
|
||||||
|
|
@ -10,7 +10,7 @@ Anthropic Messages API.
|
||||||
|
|
||||||
## Capability
|
## Capability
|
||||||
|
|
||||||
- `llm.chat@0.2.0`
|
- `llm.chat@0.3.0`
|
||||||
|
|
||||||
## Inputs
|
## Inputs
|
||||||
|
|
||||||
|
|
@ -40,6 +40,8 @@ Anthropic Messages API.
|
||||||
| `model_endpoint` | text | The endpoint the response was generated against. |
|
| `model_endpoint` | text | The endpoint the response was generated against. |
|
||||||
| `model_name` | text | The model identifier as supplied to the LLM API. |
|
| `model_name` | text | The model identifier as supplied to the LLM API. |
|
||||||
| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. |
|
| `model_digest` | text | SHA-256 digest of the served Ollama model. Empty for `openai`/`anthropic` — those APIs expose no digest; the field is left empty rather than fabricated. |
|
||||||
|
| `tokens_prompt` | text | What the prompt cost, as the backend counted it. Empty when the answer carried no count. |
|
||||||
|
| `tokens_completion` | text | What the answer cost, same rule. Ollama counts in `eval_count`, the OpenAI shape in `usage.completion_tokens`, Anthropic in `usage.output_tokens`. |
|
||||||
|
|
||||||
Together the three `model_*` outputs answer the audit
|
Together the three `model_*` outputs answer the audit
|
||||||
question: "which exact model produced this response?" — that
|
question: "which exact model produced this response?" — that
|
||||||
|
|
|
||||||
BIN
module.wasm
BIN
module.wasm
Binary file not shown.
30
module.yaml
30
module.yaml
|
|
@ -1,12 +1,12 @@
|
||||||
schema_version: 3
|
schema_version: 3
|
||||||
provider: chain
|
provider: chain
|
||||||
name: llm-chat
|
name: llm-chat
|
||||||
version: 0.2.2
|
version: 0.3.0
|
||||||
|
|
||||||
# Capability provided by this module.
|
# Capability provided by this module.
|
||||||
provides:
|
provides:
|
||||||
- capability: llm.chat
|
- capability: llm.chat
|
||||||
version: 0.2.2
|
version: 0.3.0
|
||||||
|
|
||||||
# Declared inputs in verbose v3 form. Studio renders the
|
# Declared inputs in verbose v3 form. Studio renders the
|
||||||
# description in the active locale as a tooltip on each
|
# description in the active locale as a tooltip on each
|
||||||
|
|
@ -171,6 +171,32 @@ outputs:
|
||||||
keine Digest-API bereit). Zusammen mit model_endpoint und
|
keine Digest-API bereit). Zusammen mit model_endpoint und
|
||||||
model_name beantwortet das die Audit-Frage "welches genaue
|
model_name beantwortet das die Audit-Frage "welches genaue
|
||||||
Modell hat diese Antwort erzeugt?"
|
Modell hat diese Antwort erzeugt?"
|
||||||
|
tokens_prompt:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
How many tokens the prompt cost, as the backend counted it.
|
||||||
|
Empty when the answer carried no count: a zero would read
|
||||||
|
like a measurement nobody took. The hub lifts this field
|
||||||
|
into the audit log automatically.
|
||||||
|
de: |
|
||||||
|
Wie viele Token die Anfrage gekostet hat, nach der Zählung
|
||||||
|
des Backends. Leer, wenn die Antwort keine Zählung enthielt:
|
||||||
|
eine Null würde wie eine Messung aussehen, die niemand
|
||||||
|
gemacht hat. Der Hub hebt dieses Feld selbst in den Prüfpfad.
|
||||||
|
tokens_completion:
|
||||||
|
type: text
|
||||||
|
description:
|
||||||
|
en: |
|
||||||
|
How many tokens the answer cost, same rules as
|
||||||
|
tokens_prompt. Ollama counts in `eval_count`, the OpenAI
|
||||||
|
shape in `usage.completion_tokens`, Anthropic in
|
||||||
|
`usage.output_tokens`.
|
||||||
|
de: |
|
||||||
|
Wie viele Token die Antwort gekostet hat, gleiche Regeln wie
|
||||||
|
tokens_prompt. Ollama zählt in `eval_count`, das
|
||||||
|
OpenAI-Format in `usage.completion_tokens`, Anthropic in
|
||||||
|
`usage.output_tokens`.
|
||||||
|
|
||||||
# Permissions required.
|
# Permissions required.
|
||||||
#
|
#
|
||||||
|
|
|
||||||
11
src/lib.rs
11
src/lib.rs
|
|
@ -76,11 +76,20 @@ pub fn invoke(_ctx: Context, inputs: Inputs) -> Result<Outputs, ModuleError> {
|
||||||
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
let result = crate::llm::chat_with_identity(&client, ¶ms)
|
||||||
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
.map_err(|e| llm_error_to_module_error(e, api, &endpoint, &model))?;
|
||||||
|
|
||||||
|
// The two token counts are the cost of this call, as the backend
|
||||||
|
// counted it. They stay empty where the answer carried no count,
|
||||||
|
// because an invented zero would look like a measurement.
|
||||||
|
let (tokens_prompt, tokens_completion) = match result.tokens {
|
||||||
|
Some((prompt, completion)) => (prompt.to_string(), completion.to_string()),
|
||||||
|
None => (String::new(), String::new()),
|
||||||
|
};
|
||||||
Ok(Outputs::new()
|
Ok(Outputs::new()
|
||||||
.with_text("response", result.response)
|
.with_text("response", result.response)
|
||||||
.with_text("model_endpoint", endpoint)
|
.with_text("model_endpoint", endpoint)
|
||||||
.with_text("model_name", model)
|
.with_text("model_name", model)
|
||||||
.with_text("model_digest", result.model_digest.unwrap_or_default()))
|
.with_text("model_digest", result.model_digest.unwrap_or_default())
|
||||||
|
.with_text("tokens_prompt", tokens_prompt)
|
||||||
|
.with_text("tokens_completion", tokens_completion))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn payload_text(p: &Payload) -> Option<String> {
|
fn payload_text(p: &Payload) -> Option<String> {
|
||||||
|
|
|
||||||
109
src/llm.rs
109
src/llm.rs
|
|
@ -393,6 +393,41 @@ pub fn extract_anthropic_content(body: &str) -> Result<String, LlmError> {
|
||||||
pub struct ChatWithIdentity {
|
pub struct ChatWithIdentity {
|
||||||
pub response: String,
|
pub response: String,
|
||||||
pub model_digest: Option<String>,
|
pub model_digest: Option<String>,
|
||||||
|
/// How many tokens the call cost, as the backend counted them:
|
||||||
|
/// (prompt, completion). `None` where the answer carries no
|
||||||
|
/// count, which is honest about a number nobody measured.
|
||||||
|
pub tokens: Option<(u64, u64)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The token counts of one answer, per wire format.
|
||||||
|
///
|
||||||
|
/// Three dialects, three spellings of the same two numbers: Ollama
|
||||||
|
/// counts in `prompt_eval_count` and `eval_count`, the OpenAI shape
|
||||||
|
/// in `usage.prompt_tokens` and `usage.completion_tokens`, Anthropic
|
||||||
|
/// in `usage.input_tokens` and `usage.output_tokens`. A missing
|
||||||
|
/// count is `None` rather than a zero, because zero is a
|
||||||
|
/// measurement and this is the absence of one.
|
||||||
|
pub fn extract_tokens(api: Api, body: &str) -> Option<(u64, u64)> {
|
||||||
|
let v: serde_json::Value = serde_json::from_str(body).ok()?;
|
||||||
|
let pair = match api {
|
||||||
|
Api::Ollama => (
|
||||||
|
v.get("prompt_eval_count").and_then(|n| n.as_u64()),
|
||||||
|
v.get("eval_count").and_then(|n| n.as_u64()),
|
||||||
|
),
|
||||||
|
Api::Openai | Api::Vllm => (
|
||||||
|
v.pointer("/usage/prompt_tokens").and_then(|n| n.as_u64()),
|
||||||
|
v.pointer("/usage/completion_tokens")
|
||||||
|
.and_then(|n| n.as_u64()),
|
||||||
|
),
|
||||||
|
Api::Anthropic => (
|
||||||
|
v.pointer("/usage/input_tokens").and_then(|n| n.as_u64()),
|
||||||
|
v.pointer("/usage/output_tokens").and_then(|n| n.as_u64()),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
match pair {
|
||||||
|
(None, None) => None,
|
||||||
|
(prompt, completion) => Some((prompt.unwrap_or(0), completion.unwrap_or(0))),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run one chat call in the selected wire format AND, for Ollama,
|
/// Run one chat call in the selected wire format AND, for Ollama,
|
||||||
|
|
@ -429,9 +464,11 @@ pub fn chat_with_identity<C: LlmClient>(
|
||||||
Api::Ollama => probe_model_digest(client, p),
|
Api::Ollama => probe_model_digest(client, p),
|
||||||
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
||||||
};
|
};
|
||||||
|
let tokens = extract_tokens(p.api, &response_body);
|
||||||
Ok(ChatWithIdentity {
|
Ok(ChatWithIdentity {
|
||||||
response,
|
response,
|
||||||
model_digest,
|
model_digest,
|
||||||
|
tokens,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -815,6 +852,47 @@ mod tests {
|
||||||
assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none());
|
assert!(derive_show_url("https://api.openai.com/v1/chat/completions").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Three dialects, three spellings of the same two numbers.
|
||||||
|
#[test]
|
||||||
|
fn every_dialect_reports_its_token_counts() {
|
||||||
|
let ollama = r#"{"message":{"content":"ja"},"prompt_eval_count":31,"eval_count":7}"#;
|
||||||
|
assert_eq!(extract_tokens(Api::Ollama, ollama), Some((31, 7)));
|
||||||
|
|
||||||
|
let openai = r#"{"choices":[{"message":{"content":"ja"}}],
|
||||||
|
"usage":{"prompt_tokens":31,"completion_tokens":7,"total_tokens":38}}"#;
|
||||||
|
assert_eq!(extract_tokens(Api::Openai, openai), Some((31, 7)));
|
||||||
|
assert_eq!(extract_tokens(Api::Vllm, openai), Some((31, 7)));
|
||||||
|
|
||||||
|
let anthropic = r#"{"content":[{"text":"ja"}],
|
||||||
|
"usage":{"input_tokens":31,"output_tokens":7}}"#;
|
||||||
|
assert_eq!(extract_tokens(Api::Anthropic, anthropic), Some((31, 7)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A backend that counts nothing leaves the fields empty, rather
|
||||||
|
/// than reporting a zero nobody measured.
|
||||||
|
#[test]
|
||||||
|
fn an_answer_without_counts_reports_none() {
|
||||||
|
assert_eq!(
|
||||||
|
extract_tokens(Api::Ollama, r#"{"message":{"content":"ja"}}"#),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
extract_tokens(Api::Openai, r#"{"choices":[{"message":{"content":"ja"}}]}"#),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(extract_tokens(Api::Anthropic, "not json"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One count present and the other missing still says what was
|
||||||
|
/// measured.
|
||||||
|
#[test]
|
||||||
|
fn a_half_count_is_reported_as_far_as_it_goes() {
|
||||||
|
assert_eq!(
|
||||||
|
extract_tokens(Api::Ollama, r#"{"prompt_eval_count":31}"#),
|
||||||
|
Some((31, 0))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn extract_show_digest_finds_top_level() {
|
fn extract_show_digest_finds_top_level() {
|
||||||
let body = r#"{"digest":"sha256:abc"}"#;
|
let body = r#"{"digest":"sha256:abc"}"#;
|
||||||
|
|
@ -841,6 +919,37 @@ mod tests {
|
||||||
assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef"));
|
assert_eq!(result.model_digest.as_deref(), Some("sha256:deadbeef"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The counts of the answer reach the caller, not only the
|
||||||
|
/// extractor: the cost of a call belongs in the trail.
|
||||||
|
#[test]
|
||||||
|
fn chat_with_identity_carries_the_token_counts() {
|
||||||
|
let canned_chat = r#"{"message":{"content":"ja"},
|
||||||
|
"prompt_eval_count":31,"eval_count":7}"#
|
||||||
|
.to_string();
|
||||||
|
let canned_show = r#"{"digest":"sha256:deadbeef"}"#.to_string();
|
||||||
|
let client = MockClient::new(vec![Ok(canned_show), Ok(canned_chat)]);
|
||||||
|
let p = ChatParams {
|
||||||
|
endpoint: "http://localhost:11434/api/chat",
|
||||||
|
..params(Api::Ollama)
|
||||||
|
};
|
||||||
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
|
assert_eq!(result.tokens, Some((31, 7)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A backend that counts nothing leaves the caller with nothing
|
||||||
|
/// to report, rather than with a zero.
|
||||||
|
#[test]
|
||||||
|
fn chat_with_identity_reports_no_counts_when_there_are_none() {
|
||||||
|
let canned_chat = r#"{"choices":[{"message":{"content":"ja"}}]}"#.to_string();
|
||||||
|
let client = MockClient::new(vec![Ok(canned_chat)]);
|
||||||
|
let p = ChatParams {
|
||||||
|
endpoint: "https://api.openai.com/v1/chat/completions",
|
||||||
|
..params(Api::Openai)
|
||||||
|
};
|
||||||
|
let result = chat_with_identity(&client, &p).unwrap();
|
||||||
|
assert_eq!(result.tokens, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn chat_with_identity_swallows_show_failure() {
|
fn chat_with_identity_swallows_show_failure() {
|
||||||
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();
|
let canned_chat = r#"{"message":{"content":"ok"}}"#.to_string();
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue