diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index f3b9fd7e68..697a7ecc1d 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2076,6 +2076,39 @@ mod tests { Ok(()) } + /// An engine that reports a failure in-band, as a `data:` error payload in a + /// 200 stream followed by `[DONE]`, must fail the smoke test with the + /// engine's own message, whether it comes before or after content tokens. + /// Pins `run_smoke_test`'s doc comment and `parse_openai_sse_line`. + #[test] + fn serve_smoke_test_fails_on_an_in_band_error_payload_in_a_200_stream() -> Result<()> { + for (name, body) in [ + ( + "serve-smoke-in-band-error", + "data: {\"error\":{\"message\":\"CUDA out of memory\",\"type\":\"InternalServerError\"}}\n\ndata: [DONE]\n\n", + ), + ( + "serve-smoke-in-band-error-after-content", + "data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\ndata: {\"object\":\"error\",\"message\":\"CUDA out of memory\"}\n\ndata: [DONE]\n\n", + ), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write_chunked_sse_head(stream)?; + write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", body.len(), body)?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("{name}: an in-band error payload was reported as measured"); + }; + assert!( + reason.contains("CUDA out of memory"), + "{name}: the reason dropped the engine's message" + ); + } + Ok(()) + } + /// A close-delimited stream (no chunking, no `Content-Length`) that ends /// after some tokens is deliberately `Measured`: it has no framing to cut /// short, so a clean close is indistinguishable from completion (see diff --git a/apps/rocm/src/providers/wire.rs b/apps/rocm/src/providers/wire.rs index 4892bf2866..fb90ec6863 100644 --- a/apps/rocm/src/providers/wire.rs +++ b/apps/rocm/src/providers/wire.rs @@ -460,6 +460,30 @@ fn parse_openai_sse_line(line: &str) -> Result> { } let payload = serde_json::from_str::(data) .context("failed to parse local provider SSE data")?; + // Servers such as vLLM report a mid-generation failure in-band, as a + // `{"error": ...}` payload or a bare `{"object": "error", ...}` one, and + // then still end the stream with `[DONE]`. Skipping it would present the + // text produced so far as a complete answer. Only the shapes those servers + // send count: a non-empty object or string under `error`. `null`, `false`, + // `0`, `""`, `[]` and `{}` are not an error. + let error = payload.get("error").filter(|error| match error { + serde_json::Value::Object(map) => !map.is_empty(), + serde_json::Value::String(text) => !text.is_empty(), + _ => false, + }); + let has_error = error.is_some() + || payload.get("object").and_then(serde_json::Value::as_str) == Some("error"); + if has_error { + // Each tier is resolved to a string before the next is tried, so an + // `error` object with no string `message` still yields a top-level one. + let message = error + .and_then(|error| error.get("message")) + .and_then(serde_json::Value::as_str) + .or_else(|| error.and_then(serde_json::Value::as_str)) + .or_else(|| payload.get("message").and_then(serde_json::Value::as_str)) + .unwrap_or("unknown OpenAI-compatible stream error"); + bail!("OpenAI-compatible provider stream failed: {message}"); + } let content = payload .get("choices") .and_then(serde_json::Value::as_array) diff --git a/apps/rocm/src/providers/wire/sse_emitter_proptests.rs b/apps/rocm/src/providers/wire/sse_emitter_proptests.rs index 77cabbb61a..3c597d2c84 100644 --- a/apps/rocm/src/providers/wire/sse_emitter_proptests.rs +++ b/apps/rocm/src/providers/wire/sse_emitter_proptests.rs @@ -14,7 +14,8 @@ //! not: CRLF line endings (what Python SSE servers emit by default), //! multi-byte UTF-8 content so that a split can land inside a UTF-8 sequence, //! `data:` with no space before content, a role-only delta, a usage chunk with -//! empty `choices`, a `message.content` chunk, malformed JSON, Anthropic's +//! empty `choices`, a `message.content` chunk, malformed JSON, in-band OpenAI error payloads in each shape +//! servers send, Anthropic's //! `message_start`, `content_block_start` and `error` events, a non-text //! delta, a `[DONE]` line in an Anthropic stream, lines after the end of the //! stream, and bodies whose last line is cut off before its terminator, so @@ -28,7 +29,8 @@ //! which is a panic guard only: it asserts nothing about what a line parses //! to. Its companion test checks that the generator keeps reaching every //! field-reading branch, so the guard does not go quiet on one. -//! - An ignored reproducer for #516 (in-band OpenAI error payloads). +//! - A test that an in-band OpenAI error payload fails the stream rather than +//! ending it cleanly, and that nothing after it is read. //! //! The transport readers that `local_http`'s `proptests` module covers (chunked, //! `Content-Length` and close-delimited bodies) feed only the OpenAI emitter, @@ -111,6 +113,55 @@ const OPENAI_FIXTURES: &[Fixture] = &[ &[("ok", false)], Some("failed to parse local provider SSE data"), ), + // A vLLM-style in-band error after some text, then the `[DONE]` that + // follows it: the error fails the stream with the server's message, the + // earlier text still reaches the caller, and neither the `[DONE]` nor the + // late chunk is read. + ( + "data: {\"choices\":[{\"delta\":{\"content\":\"par\"}}]}\n\ndata: {\"error\":{\"object\":\"error\",\"message\":\"CUDA out of memory\",\"type\":\"InternalServerError\",\"code\":500}}\n\ndata: [DONE]\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\n", + &[("par", false)], + Some("OpenAI-compatible provider stream failed: CUDA out of memory"), + ), + // The bare error object that older releases send, on a CRLF stream. + ( + "data: {\"object\":\"error\",\"message\":\"engine died\",\"type\":\"InternalServerError\",\"code\":500}\r\n\r\ndata: [DONE]\r\n\r\n", + &[], + Some("OpenAI-compatible provider stream failed: engine died"), + ), + // An error with no `[DONE]` after it, the last line cut off before its + // terminator: `finish()` must still fail the stream, not end it. + ( + "data: {\"choices\":[{\"delta\":{\"content\":\"par\"}}]}\n\ndata: {\"error\":{\"message\":\"boom\"}}", + &[("par", false)], + Some("OpenAI-compatible provider stream failed: boom"), + ), + // `"error": null` beside normal content is not an error. + ( + "data: {\"error\":null,\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\ndata: [DONE]\n\n", + &[("ok", false), ("", true)], + None, + ), + // `error` as a bare string. + ( + "data: {\"error\":\"model unloaded\"}\n\ndata: [DONE]\n\n", + &[], + Some("OpenAI-compatible provider stream failed: model unloaded"), + ), + // `error` with no message: a placeholder stands in for it. + ( + "data: {\"error\":{\"code\":500}}\n\n", + &[], + Some("OpenAI-compatible provider stream failed: unknown OpenAI-compatible stream error"), + ), + // An empty `error` object alone is not an error (only a non-empty object or + // string is), so the stream just ends. + ("data: {\"error\":{}}\n\n", &[("", true)], None), + // A payload that is JSON but not an object is no event and no error. + ( + "data: 42\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"ok\"}}]}\n\ndata: [DONE]\n\n", + &[("ok", false), ("", true)], + None, + ), ]; const ANTHROPIC_FIXTURES: &[Fixture] = &[ @@ -596,12 +647,12 @@ fn a_callback_error_stops_either_emitter() { /// `data: [DONE]` (and older releases yield the bare error object). Both shapes /// below are written from that handler, not captured from a live server. /// -/// Today the error line parses to "no content" and is skipped, so the caller -/// sees a normal, finished stream holding only the text produced before the -/// failure: a truncated answer presented as a complete one. The fix for #516 -/// should un-ignore this. +/// Such a payload fails the stream with the server's message, so the caller +/// never mistakes the text produced before the failure for a complete answer; +/// that text has already reached the caller's callback. Nothing after the error +/// line is read, so the `[DONE]` that follows it does not turn the failure into +/// a finish. #[test] -#[ignore = "known gap: in-band SSE error payloads are dropped and the stream reports a clean finish (#516)"] fn in_band_stream_errors_are_not_reported_as_a_clean_finish() { for error_line in [ "data: {\"error\":{\"object\":\"error\",\"message\":\"CUDA out of memory\",\"type\":\"InternalServerError\",\"code\":500}}\n\n", @@ -611,9 +662,10 @@ fn in_band_stream_errors_are_not_reported_as_a_clean_finish() { "data: {{\"choices\":[{{\"delta\":{{\"content\":\"partial\"}}}}]}}\n\n{error_line}data: [DONE]\n\n" ); let (events, error) = openai_outcome(&[body.as_bytes()]); - assert!( - error.is_some(), - "stream with an in-band error finished cleanly with {events:?}" + assert_eq!( + error.as_deref(), + Some("OpenAI-compatible provider stream failed: CUDA out of memory"), + "stream with an in-band error did not fail with its message; events: {events:?}" ); // The text produced before the failure still reaches the caller. assert_eq!( @@ -623,5 +675,76 @@ fn in_band_stream_errors_are_not_reported_as_a_clean_finish() { .collect::(), "partial" ); + assert!(events.iter().all(|e| !e.done)); + } +} + +/// The message comes from `error.message`, then a string `error`, then a +/// top-level `message`, each tried only if the one before is not a string; with +/// none the placeholder is used. +#[test] +fn in_band_error_message_falls_back_tier_by_tier() { + for (line, expected) in [ + ("{\"error\":{\"code\":500},\"message\":\"top\"}", "top"), + ( + "{\"error\":{},\"message\":\"top\",\"object\":\"error\"}", + "top", + ), + ( + "{\"error\":[1,2],\"message\":\"top\",\"object\":\"error\"}", + "top", + ), + ("{\"error\":\"plain\",\"message\":\"top\"}", "plain"), + ( + "{\"error\":{\"message\":\"inner\"},\"message\":\"top\"}", + "inner", + ), + ( + "{\"error\":{\"code\":500}}", + "unknown OpenAI-compatible stream error", + ), + ] { + let body = format!("data: {line}\n\n"); + let (_, error) = openai_outcome(&[body.as_bytes()]); + assert_eq!( + error.as_deref(), + Some(format!("OpenAI-compatible provider stream failed: {expected}").as_str()), + "wrong message for {line}" + ); } } + +/// A falsy `error` field next to real content is not a failure, so a server that +/// sends one does not have its stream failed. +#[test] +fn a_falsy_error_field_does_not_fail_the_stream() { + for falsy in ["null", "false", "0", "\"\"", "[]", "{}"] { + let body = format!( + "data: {{\"error\":{falsy},\"choices\":[{{\"delta\":{{\"content\":\"ok\"}}}}]}}\n\ndata: [DONE]\n\n" + ); + let (events, error) = openai_outcome(&[body.as_bytes()]); + assert_eq!(error, None, "error: {falsy} failed the stream"); + assert_eq!( + events + .iter() + .map(|e| e.content.as_str()) + .collect::(), + "ok" + ); + } +} + +/// After an in-band error the emitter reads no further line, even in the same +/// chunk: the caller's callback is not called again and a later error does not +/// replace the first. +#[test] +fn lines_after_an_in_band_error_are_not_read() { + let (events, error) = openai_outcome(&[ + b"data: {\"error\":{\"message\":\"first\"}}\n\ndata: {\"choices\":[{\"delta\":{\"content\":\"late\"}}]}\n\ndata: {\"error\":{\"message\":\"second\"}}\n\ndata: [DONE]\n\n", + ]); + assert!(events.is_empty(), "events after an error: {events:?}"); + assert_eq!( + error.as_deref(), + Some("OpenAI-compatible provider stream failed: first") + ); +} diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index c30bcc654a..5849bfab73 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -87,12 +87,14 @@ pub(crate) enum SmokeOutcome { /// `openai_chat_completion_probe` also allows for readiness. The status check /// is deliberately stricter than that probe's (any non-200 fails here, where /// the probe accepts anything below 500). Not detected here, and so - /// `Measured`: an error reported inside a 200 stream as a `data:` payload, a - /// 200 answer with no `data:` line at all (HTML, plain JSON, an empty body), - /// and a close-delimited stream cut after a token. + /// `Measured`: a 200 answer with no `data:` line at all (HTML, plain JSON, an + /// empty body), and a close-delimited stream cut after a token. An error + /// reported inside a 200 stream as a `data:` payload is detected and is + /// [`SmokeOutcome::Failed`] (`parse_openai_sse_line` in `providers/wire.rs`). Measured(SmokeMetrics), /// The smoke test could not complete: a transport error, an HTTP error - /// status, a parse error, a chunked or `Content-Length` stream cut short, or + /// status, a parse error, an error reported in-band as a `data:` payload in a + /// 200 stream, a chunked or `Content-Length` stream cut short, or /// a failure in the provider plumbing around the request (finding the ready /// service, writing the audit event after the stream). Carries a one-line /// reason. It does not by itself prove the engine failed. @@ -378,10 +380,11 @@ pub(crate) fn oom_memory_note( /// does any other error from the provider call, including two that are not the /// engine's fault: no ready service found when it re-lists `/v1/models`, and the /// audit-log write after the stream. A request that completes without a content -/// token is `Measured` with empty metrics, not a failure. Also `Measured`, and so -/// not detected: an error `data:` payload inside a 200 stream, a 200 answer with -/// no `data:` line at all, and a close-delimited stream cut after a token (it has -/// no framing to cut short). +/// token is `Measured` with empty metrics, not a failure. An error reported +/// in-band, as a `data:` payload inside a 200 stream, is detected and is +/// `Failed` with the server's message, even after content tokens arrived. Also +/// `Measured`, and so not detected: a 200 answer with no `data:` line at all, and +/// a close-delimited stream cut after a token (it has no framing to cut short). pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome { let request = ChatRequest { model: Some(api_model.to_owned()), diff --git a/docs/manual-testing.md b/docs/manual-testing.md index ec2b270669..38b23b27d9 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1978,9 +1978,11 @@ becomes ready: - `rocm serve --no-smoke-test` prints the plain `Deployment summary` with no smoke-test note, and re-running against an already-running service prints the `(already running)` summary with no smoke-test note. -- Some failures are deliberately not detected and still read as `ready` (an error - sent as a `data:` payload inside a `200` stream, a `200` answer with no `data:` - line, a close-delimited stream cut after a token); the full list is in +- An error the engine sends as a `data:` payload inside a `200` stream is + detected: the summary reads `started, smoke test failed` with the engine's + message. Some failures are deliberately not detected and still read as `ready` + (a `200` answer with no `data:` line, a close-delimited stream cut after a + token); the full list is in [testing.md](testing.md#serve-smoke-test-failure). - A reasoning model that spends its small token budget before emitting any content is not a failure: the summary stays `ready` with the two metric rows diff --git a/docs/testing.md b/docs/testing.md index e8cf19058e..61f5900b40 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -665,7 +665,7 @@ ignored, and its reason starts with `known gap:` and ends with the issue number: ```rust -#[ignore = "known gap: in-band SSE error payloads are dropped and the stream reports a clean finish (#516)"] +#[ignore = "known gap: the ComfyUI unpack keeps world-writable modes from the archive (#522)"] ``` The prefix is a rule, not a convention, because it is the only way to find @@ -4282,15 +4282,16 @@ What is detected, and what is not: - Detected (the summary says the smoke test failed): a transport error or timeout, an HTTP status other than 200, a parse error, and a chunked or `Content-Length` - stream cut short. -- Not detected (no failure is reported and the status reads `ready`): an engine - error sent as a `data:` payload inside a `200` stream; a `200` answer with no - `data:` line at all (HTML, plain JSON, an empty body); a close-delimited stream - cut after a token; and a complete stream with no `content` token, which is - deliberately not a failure (a reasoning model can spend its whole small budget - first, as the readiness probe also allows for). The metric rows read `n/a` when - no content token arrived first; tokens that did arrive (the close-delimited cut, - or an in-band `data:` error after content) are measured. + stream cut short, and an engine error sent as a `data:` payload inside a `200` + stream (vLLM's `{"error": ...}` or `{"object": "error", ...}`), including one + that follows content tokens. The reason shown is the server's own message. +- Not detected (no failure is reported and the status reads `ready`): a `200` + answer with no `data:` line at all (HTML, plain JSON, an empty body); a + close-delimited stream cut after a token; and a complete stream with no + `content` token, which is deliberately not a failure (a reasoning model can + spend its whole small budget first, as the readiness probe also allows for). + The metric rows read `n/a` when no content token arrived first; tokens that + did arrive in a close-delimited cut are measured. - Known limitation: the reason shown is whatever the provider call reported, and some of its errors are not the engine's, for example the `/v1/models` re-list before the request ("no ready managed service"), an unreadable services-directory @@ -4320,8 +4321,9 @@ cargo test -p rocm --bin rocm the_commands_the_failure_note_names_exist_and_pars The `serve_smoke_test` tests in [providers.rs](../apps/rocm/src/providers.rs) run the real smoke test against a local mock engine over a socket (an engine that -dies mid-answer or inside a chunk, a 500, 401 and 404, a hostile status line, a -complete stream with no content, a close-delimited stream cut after a token). Only +dies mid-answer or inside a chunk, a 500, 401 and 404, a hostile status line, an +in-band `data:` error, a complete stream with no content, a close-delimited +stream cut after a token). Only the mid-stream-death test goes on to render the summary from the real outcome; the others assert on the outcome the smoke test returns.