feat: api accepts 'vllm' as a first-class value
Some checks failed
CI / Linux x86_64 (Forgejo) (push) Failing after 2s
sign-bundle / sign (push) Failing after 1m53s

'openai' reads as the cloud company — an operator running
self-hosted vLLM should be able to write what they mean. The new
value speaks the identical OpenAI wire (a guard test asserts the
emitted request stays byte-identical to api: openai, so the alias
can never drift into a dialect) but yields vLLM-specific error
hints (vllm serve, port 8000) instead of generic OpenAI prose.
Manifest + docs name the value in DE and EN.

Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
flemming-it 2026-08-20 17:13:44 +02:00
parent 065aed4dff
commit b5ceaa3dfe
3 changed files with 55 additions and 13 deletions

View file

@ -35,13 +35,20 @@ inputs:
type: text
description:
en: |
Optional wire format: "ollama" (default), "openai"
(Chat Completions — OpenAI, vLLM, compatible servers),
or "anthropic" (Messages API). Empty = ollama.
Optional wire format: "ollama" (default), "vllm"
(self-hosted vLLM), "openai" (OpenAI or other
OpenAI-compatible servers such as LM Studio), or
"anthropic" (Messages API). vllm and openai speak the
same wire; the separate value exists so you can write
what you mean and get vLLM-specific hints. Empty = ollama.
de: |
Optionales Wire-Format: "ollama" (Default), "openai"
(Chat Completions — OpenAI, vLLM, kompatible Server)
oder "anthropic" (Messages API). Leer = ollama.
Optionales Wire-Format: "ollama" (Default), "vllm"
(selbst gehostetes vLLM), "openai" (OpenAI oder andere
OpenAI-kompatible Server wie LM Studio) oder "anthropic"
(Messages API). vllm und openai sprechen dieselbe Wire;
der eigene Wert existiert, damit Du schreibst, was Du
meinst, und vLLM-spezifische Hinweise bekommst.
Leer = ollama.
model:
type: text
description:

View file

@ -81,7 +81,12 @@ fn llm_error_to_module_error(
)),
(Api::Openai, LlmError::Http(detail)) => ModuleError::internal(format!(
"LLM endpoint {endpoint} not reachable ({detail}). Expected an \
OpenAI-compatible server (OpenAI, vLLM, ...) at a /v1/chat/completions URL."
OpenAI-compatible server at a /v1/chat/completions URL."
)),
(Api::Vllm, LlmError::Http(detail)) => ModuleError::internal(format!(
"LLM endpoint {endpoint} not reachable ({detail}). Is vLLM running? \
Start it with `vllm serve {model}`, then point `endpoint` at \
http://<host>:8000/v1/chat/completions."
)),
(Api::Anthropic, LlmError::Http(detail)) => ModuleError::internal(format!(
"LLM endpoint {endpoint} not reachable ({detail}). Expected the \
@ -105,7 +110,7 @@ fn llm_error_to_module_error(
ModuleError::invalid_input(format!("missing required input '{name}'"))
}
(_, LlmError::UnsupportedApi(raw)) => ModuleError::invalid_input(format!(
"unsupported api '{raw}' (expected: ollama, openai, anthropic)"
"unsupported api '{raw}' (expected: ollama, openai, vllm, anthropic)"
)),
}
}

View file

@ -33,7 +33,7 @@ const ANTHROPIC_VERSION: &str = "2023-06-01";
pub enum LlmError {
#[error("missing required input '{0}'")]
MissingInput(&'static str),
#[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")]
#[error("unsupported api '{0}' (expected: ollama, openai, vllm, anthropic)")]
UnsupportedApi(String),
#[error("http error: {0}")]
Http(String),
@ -48,6 +48,13 @@ pub enum LlmError {
pub enum Api {
Ollama,
Openai,
/// vLLM speaks the OpenAI wire format verbatim; it is a
/// separate variant ONLY so operators can write what they
/// mean (`api: vllm`) and get vLLM-specific error hints —
/// "openai" would read as the cloud company, not a
/// self-hosted server. Same rationale as the hub's
/// SystemLlmProvider vocabulary.
Vllm,
Anthropic,
}
@ -58,6 +65,7 @@ impl Api {
match raw.trim().to_ascii_lowercase().as_str() {
"" | "ollama" => Ok(Api::Ollama),
"openai" => Ok(Api::Openai),
"vllm" => Ok(Api::Vllm),
"anthropic" => Ok(Api::Anthropic),
other => Err(LlmError::UnsupportedApi(other.to_string())),
}
@ -92,7 +100,7 @@ pub struct ChatParams<'a> {
pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> {
let mut headers = Vec::with_capacity(2);
match p.api {
Api::Ollama | Api::Openai => {
Api::Ollama | Api::Openai | Api::Vllm => {
if !p.api_key.is_empty() {
headers.push(("Authorization", format!("Bearer {}", p.api_key)));
}
@ -261,19 +269,19 @@ pub fn chat_with_identity<C: LlmClient>(
}
let body = match p.api {
Api::Ollama => build_ollama_body(p),
Api::Openai => build_openai_body(p),
Api::Openai | Api::Vllm => build_openai_body(p),
Api::Anthropic => build_anthropic_body(p),
};
let headers = build_headers(p);
let response_body = client.post_json(p.endpoint, &body, &headers)?;
let response = match p.api {
Api::Ollama => extract_ollama_content(&response_body)?,
Api::Openai => extract_openai_content(&response_body)?,
Api::Openai | Api::Vllm => extract_openai_content(&response_body)?,
Api::Anthropic => extract_anthropic_content(&response_body)?,
};
let model_digest = match p.api {
Api::Ollama => probe_model_digest(client, p),
Api::Openai | Api::Anthropic => None,
Api::Openai | Api::Vllm | Api::Anthropic => None,
};
Ok(ChatWithIdentity {
response,
@ -717,4 +725,26 @@ mod tests {
Err(LlmError::MissingInput("prompt"))
));
}
/// Guard: `api: vllm` exists ONLY for operator clarity — the
/// request it emits (URL, body, headers) must stay identical to
/// `api: openai`. If the two ever diverge, that is a new wire
/// dialect and needs its own deliberate design, not drift.
#[test]
fn vllm_is_the_openai_wire_verbatim() {
assert_eq!(Api::parse("vllm").unwrap(), Api::Vllm);
let openai_reply = r#"{"choices":[{"message":{"role":"assistant","content":"x"}}]}"#;
let recorded: Vec<RecordedCall> = [Api::Vllm, Api::Openai]
.into_iter()
.map(|api| {
let client = MockClient::new(vec![Ok(openai_reply.to_string())]);
chat_with_identity(&client, &params(api)).unwrap();
client.calls.into_inner().pop().unwrap()
})
.collect();
assert_eq!(
recorded[0], recorded[1],
"vllm request drifted from openai"
);
}
}