feat: api accepts 'vllm' as a first-class value
'openai' reads as the cloud company — an operator running self-hosted vLLM should be able to write what they mean. The new value speaks the identical OpenAI wire (a guard test asserts the emitted request stays byte-identical to api: openai, so the alias can never drift into a dialect) but yields vLLM-specific error hints (vllm serve, port 8000) instead of generic OpenAI prose. Manifest + docs name the value in DE and EN. Signed-off-by: flemming-it <sf@flemming.it>
This commit is contained in:
parent
065aed4dff
commit
b5ceaa3dfe
3 changed files with 55 additions and 13 deletions
40
src/llm.rs
40
src/llm.rs
|
|
@ -33,7 +33,7 @@ const ANTHROPIC_VERSION: &str = "2023-06-01";
|
|||
pub enum LlmError {
|
||||
#[error("missing required input '{0}'")]
|
||||
MissingInput(&'static str),
|
||||
#[error("unsupported api '{0}' (expected: ollama, openai, anthropic)")]
|
||||
#[error("unsupported api '{0}' (expected: ollama, openai, vllm, anthropic)")]
|
||||
UnsupportedApi(String),
|
||||
#[error("http error: {0}")]
|
||||
Http(String),
|
||||
|
|
@ -48,6 +48,13 @@ pub enum LlmError {
|
|||
pub enum Api {
|
||||
Ollama,
|
||||
Openai,
|
||||
/// vLLM speaks the OpenAI wire format verbatim; it is a
|
||||
/// separate variant ONLY so operators can write what they
|
||||
/// mean (`api: vllm`) and get vLLM-specific error hints —
|
||||
/// "openai" would read as the cloud company, not a
|
||||
/// self-hosted server. Same rationale as the hub's
|
||||
/// SystemLlmProvider vocabulary.
|
||||
Vllm,
|
||||
Anthropic,
|
||||
}
|
||||
|
||||
|
|
@ -58,6 +65,7 @@ impl Api {
|
|||
match raw.trim().to_ascii_lowercase().as_str() {
|
||||
"" | "ollama" => Ok(Api::Ollama),
|
||||
"openai" => Ok(Api::Openai),
|
||||
"vllm" => Ok(Api::Vllm),
|
||||
"anthropic" => Ok(Api::Anthropic),
|
||||
other => Err(LlmError::UnsupportedApi(other.to_string())),
|
||||
}
|
||||
|
|
@ -92,7 +100,7 @@ pub struct ChatParams<'a> {
|
|||
pub fn build_headers(p: &ChatParams) -> Vec<(&'static str, String)> {
|
||||
let mut headers = Vec::with_capacity(2);
|
||||
match p.api {
|
||||
Api::Ollama | Api::Openai => {
|
||||
Api::Ollama | Api::Openai | Api::Vllm => {
|
||||
if !p.api_key.is_empty() {
|
||||
headers.push(("Authorization", format!("Bearer {}", p.api_key)));
|
||||
}
|
||||
|
|
@ -261,19 +269,19 @@ pub fn chat_with_identity<C: LlmClient>(
|
|||
}
|
||||
let body = match p.api {
|
||||
Api::Ollama => build_ollama_body(p),
|
||||
Api::Openai => build_openai_body(p),
|
||||
Api::Openai | Api::Vllm => build_openai_body(p),
|
||||
Api::Anthropic => build_anthropic_body(p),
|
||||
};
|
||||
let headers = build_headers(p);
|
||||
let response_body = client.post_json(p.endpoint, &body, &headers)?;
|
||||
let response = match p.api {
|
||||
Api::Ollama => extract_ollama_content(&response_body)?,
|
||||
Api::Openai => extract_openai_content(&response_body)?,
|
||||
Api::Openai | Api::Vllm => extract_openai_content(&response_body)?,
|
||||
Api::Anthropic => extract_anthropic_content(&response_body)?,
|
||||
};
|
||||
let model_digest = match p.api {
|
||||
Api::Ollama => probe_model_digest(client, p),
|
||||
Api::Openai | Api::Anthropic => None,
|
||||
Api::Openai | Api::Vllm | Api::Anthropic => None,
|
||||
};
|
||||
Ok(ChatWithIdentity {
|
||||
response,
|
||||
|
|
@ -717,4 +725,26 @@ mod tests {
|
|||
Err(LlmError::MissingInput("prompt"))
|
||||
));
|
||||
}
|
||||
|
||||
/// Guard: `api: vllm` exists ONLY for operator clarity — the
|
||||
/// request it emits (URL, body, headers) must stay identical to
|
||||
/// `api: openai`. If the two ever diverge, that is a new wire
|
||||
/// dialect and needs its own deliberate design, not drift.
|
||||
#[test]
|
||||
fn vllm_is_the_openai_wire_verbatim() {
|
||||
assert_eq!(Api::parse("vllm").unwrap(), Api::Vllm);
|
||||
let openai_reply = r#"{"choices":[{"message":{"role":"assistant","content":"x"}}]}"#;
|
||||
let recorded: Vec<RecordedCall> = [Api::Vllm, Api::Openai]
|
||||
.into_iter()
|
||||
.map(|api| {
|
||||
let client = MockClient::new(vec![Ok(openai_reply.to_string())]);
|
||||
chat_with_identity(&client, ¶ms(api)).unwrap();
|
||||
client.calls.into_inner().pop().unwrap()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
recorded[0], recorded[1],
|
||||
"vllm request drifted from openai"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue