diff --git a/README.md b/README.md index bb85d199b..e0e91b892 100644 --- a/README.md +++ b/README.md @@ -1262,7 +1262,15 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +quick smoke test (time to first token and approximate tokens per second). On an +interactive terminal, if the smoke test cannot complete, the summary is headed +`(smoke test failed)`, shows the status as `started, smoke test failed` instead +of `ready`, and a note gives the reason and points at +`rocm logs --service `. ROCm CLI does not stop the server, and the command +still exits 0. Which failures are detected, and which still read as `ready`, is +listed under +[Serve Smoke Test Failure](https://github.com/ROCm/rocm-cli/blob/main/docs/testing.md#serve-smoke-test-failure) +in the testing guide. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index c253f6774..83567da2b 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -19,8 +19,11 @@ e2e-test-hooks = [ ] # Test-only fault injection for the e2e suite. Enabled ONLY by the `xtask e2e` # mock-lane build (never by the release build), it compiles in a `rocm serve` -# hook that simulates a managed vLLM launch which OOM'd and never became ready, -# so the positive OOM-guidance summary can be verified on a GPU-less CI host. +# hook that simulates a managed vLLM launch, so the `rocm serve` summary can be +# verified on a GPU-less CI host: one that OOM'd and never became ready (the +# OOM guidance), or, in the hook's `ready` mode, one that became ready (the +# hook only fabricates the ready record; a mock engine supplies the failure of +# the post-launch smoke test). # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in # `service_launch/readiness.rs`. e2e-oom-fault-injection = [] diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 8b4f2b79f..9130f148d 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -560,6 +560,9 @@ rocm serve qwen --verbose --device gpu_required")] #[arg(long, conflicts_with = "managed")] verbose: bool, /// Skip the post-startup inference smoke test (time-to-first-token, tokens/sec). + /// + /// When the smoke test runs and cannot complete, the summary says so + /// and does not report the service as ready; ROCm CLI does not stop the server. #[arg(long)] no_smoke_test: bool, /// Allow binding to a non-local address. diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d35d73869..f3b9fd7e6 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1673,22 +1673,16 @@ mod tests { paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + write_chunked_sse_head(stream)?; let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; let done = "data: [DONE]\n\n"; write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", done.len(), done)?; stream.flush()?; Ok(()) - }, - ); + }); let mut ready = ManagedServiceRecord::new( &paths, @@ -1750,26 +1744,42 @@ mod tests { /// leaves behind. fn ready_service_that_dies_mid_chunked_stream( name: &str, + ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { + ready_service_answering_chat_with(name, answer_then_die_on_a_chunk_boundary) + } + + /// Sends the chunked SSE head and one content chunk, then returns, which + /// drops the stream: the client sees EOF on a chunk boundary. + fn answer_then_die_on_a_chunk_boundary(stream: &mut TcpStream) -> Result<()> { + write_chunked_sse_head(stream)?; + let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; + write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; + stream.flush()?; + Ok(()) + } + + fn write_chunked_sse_head(stream: &mut TcpStream) -> Result<()> { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + )?; + Ok(()) + } + + /// A ready local service whose engine answers the chat request with + /// `respond` (after answering any `/v1/models` listing itself). + fn ready_service_answering_chat_with( + name: &str, + mut respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { let (root, paths) = temp_app_paths(name); paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; - let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; - write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; - stream.flush()?; - // Returning drops the stream: the client sees EOF on a chunk boundary. - Ok(()) - }, - ); + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + respond(stream) + }); let mut ready = ManagedServiceRecord::new( &paths, "svc-ready", @@ -1888,30 +1898,31 @@ mod tests { } /// `rocm serve`'s post-launch smoke test is the CLI's streaming consumer. An - /// engine that dies mid-answer must leave the deployment summary showing no - /// measurements (`n/a`) instead of hanging the command or reporting - /// throughput for an answer that never finished. + /// engine that dies mid-answer must make the smoke test fail, and the + /// deployment summary built from that outcome must say so (heading, status + /// and reason, metrics `n/a`) instead of hanging the command, presenting the + /// service as ready, or reporting throughput for an answer that never + /// finished. #[test] - fn serve_smoke_test_reports_no_metrics_when_engine_dies_mid_chunked_stream() -> Result<()> { - let (root, paths, server) = - ready_service_that_dies_mid_chunked_stream("serve-smoke-chunked-truncated")?; - let (sender, receiver) = mpsc::channel(); - thread::spawn(move || { - let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); - }); - let (outcome, served) = client_then_server(&receiver, server); - fs::remove_dir_all(root).ok(); - - let metrics = outcome.map_err(|_| { - anyhow::anyhow!( - "the serve smoke test never returned after the engine closed the connection" - ) - })?; - // Empty metrics are also what a smoke test that never reached the - // engine returns, so this is what makes the assertion below mean + fn serve_smoke_test_fails_and_summary_says_so_when_engine_dies_mid_chunked_stream() -> Result<()> + { + // `smoke_outcome_against` also checks the engine side served its + // request: a failed outcome is also what a smoke test that never reached + // the engine returns, so that is what makes the assertion below mean // "the engine died mid-answer" rather than "nothing happened". - served.transpose()?; - assert_eq!(metrics, crate::serve_summary::SmokeMetrics::default()); + let smoke = smoke_outcome_against( + "serve-smoke-chunked-truncated", + answer_then_die_on_a_chunk_boundary, + )?; + // The asserts below do not interpolate the outcome or the rendered + // summary: `DeploymentSummary` carries an API key field, and echoing it + // trips CodeQL's `rust/cleartext-logging`. + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke.clone() else { + anyhow::bail!("a stream that died mid-answer was reported as measured"); + }; + // The reason is the provider's own truncation message, not just any text. + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); + let (metrics, smoke_failure) = smoke.into_parts(); let rendered = crate::serve_summary::render_summary(&crate::serve_summary::DeploymentSummary { engine: "vllm".to_owned(), @@ -1923,11 +1934,170 @@ mod tests { status: "ready".to_owned(), already_running: false, metrics, + smoke_failure, api_key: None, notes: Vec::new(), }); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("started, smoke test failed")); + assert!(!rendered.contains(" ready\n")); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains(&reason)); + assert!(rendered.contains("`rocm logs --service svc-ready`")); + Ok(()) + } + + /// Runs the smoke test against a ready service whose engine answers the chat + /// request with `respond`, and returns the outcome. + fn smoke_outcome_against( + name: &str, + respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, + ) -> Result { + let (root, paths, server) = ready_service_answering_chat_with(name, respond)?; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); + }); + let (outcome, served) = client_then_server(&receiver, server); + fs::remove_dir_all(root).ok(); + let smoke = outcome.map_err(|_| anyhow::anyhow!("the smoke test never returned"))?; + served.transpose()?; + Ok(smoke) + } + + /// A chat request that completes without error is not a failed smoke test + /// just because no `content` token arrived: a reasoning model can spend its + /// small token budget on `reasoning_content` first, and the readiness probe + /// (`openai_chat_completion_probe`) already treats such a server as serving. + /// The metrics stay empty (`n/a`) and no failure is recorded. + #[test] + fn serve_smoke_test_is_not_a_failure_when_a_complete_stream_carries_no_content() -> Result<()> { + for (name, body) in [ + ( + "serve-smoke-reasoning-only", + "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"thinking\"}}]}\n\ndata: [DONE]\n\n", + ), + ("serve-smoke-done-only", "data: [DONE]\n\n"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write_chunked_sse_head(stream)?; + write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", body.len(), body)?; + stream.flush()?; + Ok(()) + })?; + assert!( + smoke + == crate::serve_summary::SmokeOutcome::Measured( + crate::serve_summary::SmokeMetrics::default() + ), + "a complete stream with no content token was not a measured, empty outcome" + ); + let (metrics, smoke_failure) = smoke.into_parts(); + assert!(smoke_failure.is_none()); + assert!(metrics == crate::serve_summary::SmokeMetrics::default()); + } + Ok(()) + } + + /// The reason in the summary is built from the engine's own words (here its + /// HTTP status line), so it must reach the terminal as one clean line: no + /// escape sequence, no control or format character, but still the status. + #[test] + fn serve_smoke_test_failure_reason_carries_no_terminal_control_sequences() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-hostile-status", |stream| { + write!( + stream, + "HTTP/1.1 500 \u{1b}[31mBoom\u{7}\u{202e}x\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP 500 was not reported as a failure"); + }; + assert!(reason.contains("500")); + assert!(reason.contains("Boom")); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format), + "the failure reason still carries a control or format character" + ); + Ok(()) + } + + /// An engine that dies *inside* a chunk leaves an error with a cause chain + /// (the truncation message wrapping the failed read). The summary's reason + /// must keep the cause, not just the outermost message. + #[test] + fn serve_smoke_test_failure_reason_keeps_the_cause_chain() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-dies-inside-chunk", |stream| { + write_chunked_sse_head(stream)?; + // Promise 0x64 bytes, send three, close. + write!(stream, "64\r\nabc")?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an engine that died inside a chunk was not reported as a failure"); + }; + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); + assert!( + reason.contains("failed to read local provider SSE body"), + "the reason lost the cause behind the truncation message" + ); + Ok(()) + } + + /// Any HTTP error status from the engine is a failed smoke test, with the + /// status in the reason: the check is stricter than the readiness probe's + /// "anything below 500", so a 4xx fails here too. + #[test] + fn serve_smoke_test_fails_on_any_non_200_status() -> Result<()> { + for (name, status_line, code) in [ + ("serve-smoke-http-500", "500 Internal Server Error", "500"), + ("serve-smoke-http-401", "401 Unauthorized", "401"), + ("serve-smoke-http-404", "404 Not Found", "404"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write!( + stream, + "HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP error status was not reported as a failure"); + }; + assert!(reason.contains(code), "the reason lost the status code"); + } + Ok(()) + } + + /// A close-delimited stream (no chunking, no `Content-Length`) that ends + /// after some tokens is deliberately `Measured`: it has no framing to cut + /// short, so a clean close is indistinguishable from completion (see + /// `read_close_delimited_sse_body`). Pins `run_smoke_test`'s doc comment. + #[test] + fn serve_smoke_test_measures_a_close_delimited_stream_cut_after_a_token() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-close-delimited", |stream| { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n" + )?; + write!( + stream, + "data: {{\"choices\":[{{\"delta\":{{\"content\":\"hi\"}}}}]}}\n\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Measured(metrics) = smoke else { + anyhow::bail!("a close-delimited stream cut after a token was reported as failed"); + }; + assert!(metrics.ttft.is_some()); Ok(()) } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index cc1da7343..f359699db 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -864,8 +864,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // (probe returns `None`). The E2E-only // backend-failure scenario bypasses this host precondition so the black-box // test reaches Lemonade's backend boundary without real GPU hardware, and the - // E2E-only OOM-launch fault injection likewise stands in for the GPU it does - // not have. + // E2E-only launch fault injection (either mode) likewise stands in for the + // GPU it does not have. // // The scripted free-space override opens the same waiver: it exists to refuse a // Lemonade install on a drive that cannot hold it, which `rocm serve` only @@ -1193,20 +1193,23 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }; let mut spinner = spinner.into_inner(); // Not when the launch was fabricated. `simulate_oom_managed_launch` - // writes a `starting` record for a process that never existed, so the - // helper would be left watching a pid that will never appear, with - // nothing in the scenario's teardown to stop it: observed as `rocm - // daemon` processes outliving their e2e checkout by days, still - // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for + // writes a record (`starting`, or `ready` in its `ready` mode) for a + // process that never existed, so the helper would be left watching a pid + // that will never appear, with nothing in the scenario's teardown to stop + // it: observed as `rocm daemon` processes outliving their e2e checkout by + // days, still carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for // supervising a real spawned server, and there is none here. // // That — not ordering — is the whole justification. This suppression // runs *before* the OOM note is computed in the `summary_mode` block // below, so the previous claim here, that the note "is computed before // it would have run", was backwards and could not have excused it. - // serve-40's `no background automation daemon was left behind` is what - // holds the branch: deleting it left the scenario green with a `rocm - // daemon` still running. + // The `no background automation daemon was left behind` step of the + // `@id:serve-oom-launch-memory-guidance` and + // `@id:serve-smoke-failure-summary` scenarios records that none was + // started but does not discriminate this branch: the helper starts only + // when a watcher is enabled, and the isolated config those scenarios use + // enables none, so removing the branch leaves them green. // // `e2e_simulate_oom_launch` is a `const fn` returning `false` without // the test-only feature, so this branch folds away in shipped builds. @@ -1236,16 +1239,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Best-effort inference smoke test, on by default (opt out with // `--no-smoke-test`). Only meaningful for a freshly-ready server we // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { - spinner.set_label("Running smoke test…"); - // The local provider resolves the endpoint key from the per-service - // owner-only key file by service id, so the smoke test authenticates - // against a protected public endpoint without threading the secret - // through here. - serve_summary::run_smoke_test(&paths, resolve.served_model_name()) - } else { - serve_summary::SmokeMetrics::default() - }; + let (metrics, smoke_failure) = + if !no_smoke_test && !report.already_running && report.status == "ready" { + spinner.set_label("Running smoke test…"); + // The local provider resolves the endpoint key from the per-service + // owner-only key file by service id, so the smoke test authenticates + // against a protected public endpoint without threading the secret + // through here. + serve_summary::run_smoke_test(&paths, resolve.served_model_name()).into_parts() + } else { + (serve_summary::SmokeMetrics::default(), None) + }; spinner.clear(); let mut notes = collect_serve_notes( @@ -1314,6 +1318,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { status: report.status.clone(), already_running: report.already_running, metrics, + smoke_failure, api_key: launched_key, notes, }; diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index cd0ab34bc..c30bcc654 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -32,7 +32,9 @@ const SMOKE_PROMPT: &str = "Reply with a short one-sentence greeting."; const SMOKE_MAX_TOKENS: u32 = 32; /// Best-effort metrics measured against a freshly-started server. Every field is -/// optional: any probe failure leaves it `None` and the summary renders `n/a`. +/// optional: `None` renders as `n/a`. They are empty when the smoke test was +/// skipped, failed (see [`SmokeOutcome::Failed`]), or completed without a content +/// token (a reasoning model can spend its whole token budget before emitting any). #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SmokeMetrics { /// Time from request send to the first streamed content token. @@ -62,6 +64,11 @@ pub(crate) struct DeploymentSummary { /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, pub metrics: SmokeMetrics, + /// Why the post-launch smoke test failed, as one sanitised line. `None` when + /// the test passed or was not run. When `Some` (and `already_running` is + /// false, which the renderer checks) the summary is not presented as ready; + /// an already-running service never ran a smoke test and ignores it. + pub smoke_failure: Option, /// The endpoint API key for a freshly launched *public* server, shown once as /// secure client configuration. `None` for loopback binds (no auth) and for an /// already-running service (whose key was delivered at its own launch). @@ -70,6 +77,70 @@ pub(crate) struct DeploymentSummary { pub notes: Vec, } +/// Result of the post-launch smoke test. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum SmokeOutcome { + /// The request completed with no transport error, no HTTP error status and + /// no truncated stream; the metrics are what was measured. They are empty + /// when no content token arrived, which is not a failure: a reasoning model + /// can spend its small token budget before emitting any, as + /// `openai_chat_completion_probe` also allows for readiness. The status check + /// is deliberately stricter than that probe's (any non-200 fails here, where + /// the probe accepts anything below 500). Not detected here, and so + /// `Measured`: an error reported inside a 200 stream as a `data:` payload, a + /// 200 answer with no `data:` line at all (HTML, plain JSON, an empty body), + /// and a close-delimited stream cut after a token. + Measured(SmokeMetrics), + /// The smoke test could not complete: a transport error, an HTTP error + /// status, a parse error, a chunked or `Content-Length` stream cut short, or + /// a failure in the provider plumbing around the request (finding the ready + /// service, writing the audit event after the stream). Carries a one-line + /// reason. It does not by itself prove the engine failed. + Failed(String), +} + +impl SmokeOutcome { + /// Splits the outcome into the summary's `metrics` and `smoke_failure` fields. + pub(crate) fn into_parts(self) -> (SmokeMetrics, Option) { + match self { + Self::Measured(metrics) => (metrics, None), + Self::Failed(reason) => (SmokeMetrics::default(), Some(reason)), + } + } +} + +/// Longest failure reason kept for display. +const SMOKE_FAILURE_MAX_CHARS: usize = 200; + +/// Reduces an error message to one printable line. On the smoke-test path the +/// engine-controlled text in it is the HTTP status line, reason phrase included +/// (the local provider reports `local provider returned {status_line}`, not the +/// response body), and the chunk-size line it quotes when a chunked body is +/// malformed (`invalid local provider chunk size`), so it is untrusted. Terminal +/// escape sequences and any control or Unicode format character (a bidi override, +/// a zero-width mark) are dropped, line breaks and runs of whitespace become a +/// single space, backticks become apostrophes (the note quotes its own command in +/// backticks, and the reason must not add a second one), and the result is +/// length-capped. +fn one_line_reason(raw: &str) -> String { + // The escape scanner, not this function, decides where a line break falls: + // turning breaks into spaces first would let a cut-off escape sequence take + // the next real character as an intermediate byte and swallow it. + let collapsed = rocm_core::terminal::rendered_lines(raw) + .join(" ") + .split_whitespace() + .collect::>() + .join(" ") + .replace('`', "'"); + if collapsed.chars().count() > SMOKE_FAILURE_MAX_CHARS { + let mut cut: String = collapsed.chars().take(SMOKE_FAILURE_MAX_CHARS).collect(); + cut.push_str("..."); + cut + } else { + collapsed + } +} + /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not /// produce a first token. pub(crate) fn format_ttft(ttft: Option) -> String { @@ -102,8 +173,11 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { // note must make that visibly different from a healthy deployment so the // summary is never mistaken for success. let ready = summary.status == "ready"; + let smoke_failed = summary.smoke_failure.is_some() && !summary.already_running; let heading = if summary.already_running { "Deployment summary (already running)" + } else if smoke_failed { + "Deployment summary (smoke test failed)" } else if ready { "Deployment summary" } else { @@ -132,7 +206,14 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { .collect(); let rows: Vec<(&str, String)> = [ vec![ - ("status", summary.status.clone()), + ( + "status", + if smoke_failed { + "started, smoke test failed".to_owned() + } else { + summary.status.clone() + }, + ), ("engine", summary.engine.clone()), ], model_rows, @@ -189,6 +270,16 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { summary.service_id ); } + if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { + // The server answered its health check, but the smoke test could not + // complete, so it is not known to serve. Say why the metrics rows are + // empty, hedged: the reason may be rocm's own plumbing, not the engine. + let _ = writeln!( + out, + " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`", + id = summary.service_id + ); + } for note in &summary.notes { let _ = writeln!(out, " note: {note}"); } @@ -280,10 +371,18 @@ pub(crate) fn oom_memory_note( } /// Run a single small chat completion against the just-started local server and -/// measure time-to-first-token and generation throughput. Best-effort: any error -/// (server not OpenAI-compatible, refused, timed out) yields empty metrics rather -/// than failing the launch. -pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics { +/// measure time-to-first-token and generation throughput. Best-effort: a request +/// that errors (refused, an HTTP error status other than 200, timed out) or +/// whose stream is a chunked or `Content-Length` body cut short yields +/// [`SmokeOutcome::Failed`] with the reason, rather than failing the launch. So +/// does any other error from the provider call, including two that are not the +/// engine's fault: no ready service found when it re-lists `/v1/models`, and the +/// audit-log write after the stream. A request that completes without a content +/// token is `Measured` with empty metrics, not a failure. Also `Measured`, and so +/// not detected: an error `data:` payload inside a 200 stream, a 200 answer with +/// no `data:` line at all, and a close-delimited stream cut after a token (it has +/// no framing to cut short). +pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome { let request = ChatRequest { model: Some(api_model.to_owned()), messages: vec![ChatMessage { @@ -316,11 +415,16 @@ pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics }, ); - if result.is_err() { - return SmokeMetrics::default(); + if let Err(err) = result { + return SmokeOutcome::Failed(one_line_reason(&format!("{err:#}"))); } - compute_metrics(start, first_token, last_token, token_chunks) + SmokeOutcome::Measured(compute_metrics( + start, + first_token, + last_token, + token_chunks, + )) } /// Pure metric math, split out so it can be unit-tested without a live server. @@ -466,6 +570,7 @@ mod tests { ttft: Some(Duration::from_millis(180)), gen_tps: Some(42.15), }, + smoke_failure: None, api_key: None, notes: Vec::new(), } @@ -1160,4 +1265,174 @@ mod tests { "an OOM line beyond the shared tail budget must not surface the note: {notes:?}" ); } + + fn failed_summary(reason: &str) -> DeploymentSummary { + let mut summary = base_summary(); + summary.metrics = SmokeMetrics::default(); + summary.smoke_failure = Some(reason.to_owned()); + summary + } + + #[test] + fn a_failed_smoke_test_is_not_presented_as_ready_and_says_why() { + let rendered = render_summary(&failed_summary("connection closed mid-response")); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("status started, smoke test failed")); + assert!(!rendered.contains("status ready")); + assert!(rendered.contains("time to first token n/a")); + assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains("connection closed mid-response")); + // That the named commands exist and parse is + // `the_commands_the_failure_note_names_exist_and_parse`. + assert!(rendered.contains("check `rocm logs --service vllm-qwen-1720000000`")); + // The hedge: the note must not claim the server is broken, only that it + // may be unable to serve. + assert!(rendered.contains("; the server may not be able to serve requests")); + } + + #[test] + fn a_passing_or_skipped_smoke_test_adds_no_failure_note() { + let passed = render_summary(&base_summary()); + assert!(passed.starts_with("Deployment summary\n")); + assert!(!passed.contains("smoke test")); + // `--no-smoke-test`: default metrics, no failure recorded. + let mut skipped = base_summary(); + skipped.metrics = SmokeMetrics::default(); + let skipped = render_summary(&skipped); + assert!(skipped.starts_with("Deployment summary\n")); + assert!(!skipped.contains("smoke test")); + assert!(skipped.contains("status ready")); + } + + #[test] + fn an_already_running_summary_ignores_a_smoke_failure() { + let mut summary = failed_summary("boom"); + summary.already_running = true; + let rendered = render_summary(&summary); + assert!(rendered.starts_with("Deployment summary (already running)\n")); + assert!(!rendered.contains("smoke test")); + } + + #[test] + fn smoke_outcome_splits_into_summary_fields() { + let metrics = SmokeMetrics { + ttft: Some(Duration::from_millis(5)), + gen_tps: None, + }; + assert_eq!( + SmokeOutcome::Measured(metrics.clone()).into_parts(), + (metrics, None) + ); + assert_eq!( + SmokeOutcome::Failed("x".to_owned()).into_parts(), + (SmokeMetrics::default(), Some("x".to_owned())) + ); + } + + #[test] + fn failure_reason_is_one_line_without_control_characters_and_capped() { + let reason = one_line_reason("HTTP 500\r\n{\"error\":\t\"boom\"}\u{1b}[31m red\u{7}\n\n"); + assert!( + !reason.chars().any(char::is_control), + "a control character survived" + ); + assert_eq!(reason, "HTTP 500 {\"error\": \"boom\"} red"); + let long = one_line_reason(&"x".repeat(5000)); + assert_eq!(long.chars().count(), SMOKE_FAILURE_MAX_CHARS + 3); + assert!(long.ends_with("...")); + assert_eq!(one_line_reason("short"), "short"); + } + + #[test] + fn failure_reason_cap_holds_at_its_boundary() { + // Exactly the cap is kept whole; one more is cut and marked. + let at_cap = "y".repeat(SMOKE_FAILURE_MAX_CHARS); + assert!( + one_line_reason(&at_cap) == at_cap, + "a reason of exactly the cap must be kept unchanged" + ); + let over = "y".repeat(SMOKE_FAILURE_MAX_CHARS + 1); + assert!( + one_line_reason(&over) == format!("{at_cap}..."), + "a reason one character over the cap must be cut to the cap plus an ellipsis" + ); + } + + #[test] + fn failure_reason_cannot_add_a_backtick_quoted_command_to_the_note() { + // A chunk-size line the engine sent is quoted in the provider's error. + let reason = one_line_reason("invalid local provider chunk size `run `rocm stop` now`"); + assert!(!reason.contains('`')); + assert_eq!( + reason, + "invalid local provider chunk size 'run 'rocm stop' now'" + ); + let rendered = render_summary(&failed_summary(&reason)); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + // Constant message: `note` derives from a type that carries an API key. + assert!( + note.split('`').skip(1).step_by(2).count() == 1, + "the note must hold exactly one backtick-quoted command" + ); + } + + #[test] + fn failure_reason_keeps_the_text_after_a_cut_off_escape_sequence() { + // A capture cut mid-sequence must not eat the first real character after + // it: the line break has to reach the escape scanner as a break, not as + // a space it would read as part of the sequence. + for (raw, expected) in [ + ("\u{1b}[31\nHTTP 500", "HTTP 500"), + ("\u{1b}\nHTTP 500", "HTTP 500"), + ("\u{1b}[31mHTTP 500", "HTTP 500"), + ("\u{1b}[31m\nHTTP 500", "HTTP 500"), + ("HTTP\r\n500", "HTTP 500"), + ] { + assert!( + one_line_reason(raw) == expected, + "an escape sequence in the engine's message ate or kept the wrong text" + ); + } + } + + #[test] + fn failure_reason_drops_unicode_format_characters() { + // U+202E (right-to-left override) reorders how the rest of the line + // renders; U+200B and U+FEFF are invisible; U+2028 breaks the row. None + // is matched by `char::is_control`. + let reason = one_line_reason("evil\u{202e}txt\u{200b}.\u{feff}exe\u{2028}next"); + assert_eq!(reason, "eviltxt.exe next"); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format) + ); + } + + #[test] + fn the_commands_the_failure_note_names_exist_and_parse() { + use clap::Parser as _; + let rendered = render_summary(&failed_summary("boom")); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + let commands: Vec<&str> = note.split('`').skip(1).step_by(2).collect(); + // Constant messages only: `commands` is derived from `render_summary`, + // whose input type carries an API key field, and echoing that output + // trips CodeQL's `rust/cleartext-logging`. + assert!( + commands == ["rocm logs --service vllm-qwen-1720000000"], + "the failure note must name exactly the log command" + ); + for command in commands { + assert!( + crate::Cli::try_parse_from(command.split_whitespace()).is_ok(), + "a command named in the failure note is not one rocm accepts" + ); + } + } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index ae5c2fdb5..7d0e49106 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -42,8 +42,8 @@ //! failed launch, and the reuse refusals. //! - `spawn`: the detached engine child, its process and stdio plumbing, and //! `ManagedLaunchReport`. -//! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection -//! seam. +//! - `readiness`: waiting on a service, its liveness, and the fault-injection +//! seam (an OOM launch, or a ready launch). //! - `start`: the background launch path and its plain output. //! - `helper_autostart`: starting the background automation helper. //! - `attach`: the attached run and its log streaming. diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 1e14c8559..c10a29eea 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -4,12 +4,13 @@ //! Waiting on a launched or already-live managed service: the readiness budget, //! liveness of the tracked process, how a wait ends, and the end-to-end OOM -//! fault-injection seam that fabricates a launch that never became ready. +//! fault-injection seam that fabricates a launch: one that never became ready +//! (it ran out of GPU memory), or, in the `ready` mode, one that became ready. // There is deliberately no `use std::fs`, `use std::thread` or // `use rocm_core::process_is_running` here: this file names them by full path. // Their use sites sit under different `cfg` combinations (the Linux-only zombie -// probe, the OOM fault-injection seam, one Linux-only test), so any single import +// probe, the fault-injection seam, one Linux-only test), so any single import // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. @@ -180,9 +181,22 @@ pub(super) fn record_wait_outcome( /// out-of-memory traceback to the log it owns, and never became ready — so the /// positive OOM-guidance summary path can be verified on a GPU-less CI host, /// where a real launch cannot run. See [`simulate_oom_managed_launch`]. +/// +/// The value `ready` selects the one other mode: the fabricated launch is a +/// managed vLLM launch that *did* become ready (nothing to blame on memory), so a +/// scenario can point its port at a mock engine and watch what the summary does +/// when the post-launch smoke test then fails. Both values open the same seams +/// (the GPU pre-flight waiver and the background-helper suppression). #[cfg(feature = "e2e-oom-fault-injection")] pub(crate) fn e2e_simulate_oom_launch() -> bool { - std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH") + .is_some_and(|value| value == "1" || value == "ready") +} + +/// Whether the armed hook is in its `ready` mode (see [`e2e_simulate_oom_launch`]). +#[cfg(feature = "e2e-oom-fault-injection")] +fn e2e_simulate_ready_launch() -> bool { + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "ready") } /// Release builds carry no fault-injection switch: the seam compiles out @@ -193,12 +207,18 @@ pub(crate) const fn e2e_simulate_oom_launch() -> bool { } /// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a -/// managed vLLM launch that spawned, OOM'd, and never became ready, then return -/// the launch report the summary path would see for it — `already_running: false` -/// (this invocation launched it), `status: "starting"` (it failed to become -/// ready), and a `log_path` whose contents carry a real allocator OOM signature. -/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is -/// spawned. Compiled out of shipped binaries. +/// managed vLLM launch, then return the launch report the summary path would see +/// for it. `already_running` is false either way (this invocation launched it) +/// and no process is spawned. Compiled out of shipped binaries. +/// +/// - Default mode (`ROCM_E2E_SIMULATE_OOM_LAUNCH=1`): the launch OOM'd and never +/// became ready — `status: "starting"` and a `log_path` whose contents carry a +/// real allocator OOM signature. +/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob +/// guidance. +/// - `ready` mode: the launch became ready — `status: "ready"`, the record marked +/// inference-verified, and `log_path: None` (nothing failed to start), so the +/// caller's smoke test runs against whatever answers on `host:port`. #[cfg(feature = "e2e-oom-fault-injection")] #[allow(clippy::too_many_arguments)] pub(super) fn simulate_oom_managed_launch( @@ -231,6 +251,25 @@ pub(super) fn simulate_oom_managed_launch( Some(device_policy_name(device_policy).to_owned()), ); record.gpu_indices = gpu_indices.to_vec(); + if e2e_simulate_ready_launch() { + // The `ready` mode: a launch that came up. The record is what the local + // provider needs to send the smoke test to `host:port`; no log is written + // because nothing here failed to start. + record.status = "ready".to_owned(); + // Load-bearing: without it the smoke test's `/v1/models` listing probe + // fails and the reason becomes "no ready managed service". + record.inference_verified_at_unix_ms = Some(1); + record.write()?; + return Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url: format!("{}/v1", format_http_base_url(host, port)), + status: "ready".to_owned(), + already_running: false, + child_pid: None, + log_path: None, + manifest_path: Some(record.manifest_path), + }); + } record.status = "starting".to_owned(); record.write()?; // A real allocator OOM signature in the log this invocation owns, so the diff --git a/docs/architecture.md b/docs/architecture.md index 73a775da2..8d7c17a37 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,7 +120,7 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | | [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. | relocation | -| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the OOM fault-injection seam for end-to-end tests. | relocation | +| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. | relocation | | [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | | [`service_launch/test_support.rs`](../apps/rocm/src/service_launch/test_support.rs) | Test fixtures shared by more than one `service_launch/` child. | flat | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c30a23798..ec2b27066 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1941,3 +1941,47 @@ Expected result: read (N of M tokens). Try a more specific question, or a model with a larger context.` — and `rocm-cli.log` should still carry the provider's own body with `exceed_context_size_error` in it. + +## 16. Serve Smoke Test Failure + +Needs any engine `rocm serve` can start (the behaviour is not vLLM-specific) and +an interactive terminal: the smoke test and its summary run only there. The +behaviour under test is described in +[the README](../README.md#model-serving); the automated coverage is listed in +[testing.md](testing.md#serve-smoke-test-failure). + +The repeatable route is the E2E scenario, which needs no GPU but runs on Linux +and WSL only (`@requires-os:linux`): + +```bash +cargo xtask e2e -- -n "serve-60" +``` + +(`@id:serve-smoke-failure-summary`.) + +By hand, a failing first chat request is hard to provoke (the engine has to pass +its readiness probe and then fail or die on the first chat request), so check the +points below when you see it, for example after an engine is killed right as it +becomes ready: + +- The heading reads `Deployment summary (smoke test failed)`, not + `Deployment summary`. +- The `status` row reads `started, smoke test failed`, not `ready`. +- `time to first token` and `throughput (approx)` read `n/a`, and a `note:` line + gives a one-line reason and names `rocm logs --service `. +- The command exits 0 and ROCm CLI does not stop the server. If the engine died + there is no server left: `rocm logs --service ` still shows its output, and + the next `rocm services list` finds the engine gone and records the service as + `stopped` (or `failed` for AIM), so it drops out of the default list and shows only under + `rocm services list --all`. If the engine is still up and only the chat request + failed, the service stays listed and its recorded status stays `ready`. +- `rocm serve --no-smoke-test` prints the plain `Deployment summary` with + no smoke-test note, and re-running against an already-running service prints + the `(already running)` summary with no smoke-test note. +- Some failures are deliberately not detected and still read as `ready` (an error + sent as a `data:` payload inside a `200` stream, a `200` answer with no `data:` + line, a close-delimited stream cut after a token); the full list is in + [testing.md](testing.md#serve-smoke-test-failure). +- A reasoning model that spends its small token budget before emitting any + content is not a failure: the summary stays `ready` with the two metric rows + `n/a` and no smoke-test note. diff --git a/docs/testing.md b/docs/testing.md index db206b25d..2271a188f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4209,7 +4209,8 @@ whether a scenario may run at all (see the comment at a **prebuilt** `ROCM_CLI_BINARY`: the hook is compiled out there, the `@requires-oom-fault-injection` skip no longer applies, and the scenario fails for the wrong reason instead of being reported as skipped. Do not select it by -name on that route. Note also that `-n` matches the scenario *name* +name on that route, nor any other `@requires-oom-fault-injection` scenario +(`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so passing an id to it silently selects nothing. @@ -4220,14 +4221,16 @@ passing an id to it silently selects nothing. ### The OOM fault-injection capability -serve-41 needs a managed launch that really failed on VRAM, which a GPU-less CI +The `@id:serve-oom-launch-memory-guidance` scenario needs a managed launch that really failed on VRAM, which a GPU-less CI host cannot produce. It is therefore driven by a test-only fault-injection hook: the `rocm` crate feature `e2e-oom-fault-injection`, armed at run time by `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`, makes `rocm serve` fabricate the on-disk state of a managed vLLM launch that spawned, wrote an allocator OOM traceback to the -log it owns, and never became ready. No process is spawned, and the seam -compiles out entirely without the feature, so a build that does not enable it -behaves exactly as the shipped binary does. +log it owns, and never became ready. `@id:serve-smoke-failure-summary` arms the same hook with +`ROCM_E2E_SIMULATE_OOM_LAUNCH=ready`, which fabricates a launch that did become +ready, so a mock engine can then fail its smoke test. No process is spawned, and +the seam compiles out entirely without the feature, so a build that does not +enable it behaves exactly as the shipped binary does. Only `cargo xtask e2e` enables it, by building the binary under test with `--features "rocm/e2e-test-hooks rocm/e2e-oom-fault-injection"`, and it is the @@ -4240,7 +4243,7 @@ was handed a prebuilt binary. Consequently, every lane that runs a **prebuilt** `ROCM_CLI_BINARY` — the self-hosted lanes in [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml) and the [`nightly.yml`](../.github/workflows/nightly.yml) lanes — has the hook -compiled out and resolves serve-41 to a reported **skip**, not a silent pass. It +compiled out and resolves both of those scenarios to a reported **skip**, not a silent pass. It gates PRs on the GitHub-hosted mock lane, which lets `cargo xtask e2e` build the binary itself. The prebuilt lanes' `--features` floor is asserted as a superset check rather than an exact match for this reason; see the test notes on @@ -4250,11 +4253,99 @@ check rather than an exact match for this reason; see the test notes on Be precise about *which* gate produces the skip, because the hook is not always the one that decides. On the Windows lifecycle lane in [`ci.yml`](../.github/workflows/ci.yml) the hook is compiled out too, but -serve-41 is never selected there at all: that lane runs `E2E_ONLY_LIFECYCLE` and -the scenario carries `@requires-os:linux`. On the GPU lanes it is +both scenarios are never selected there at all: that lane runs +`E2E_ONLY_LIFECYCLE` and the scenarios carry `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. +## Serve Smoke Test Failure + +After a fresh launch becomes ready, the `rocm serve` interactive summary sends +one small chat request. When that request cannot complete the summary is headed +`Deployment summary (smoke test failed)`, shows the status as +`started, smoke test failed` instead of `ready`, and a hedged note gives the +reason and names `rocm logs --service `. See +[the README](../README.md#model-serving) for the user-facing contract. The +launch itself stays best-effort: exit code and service record do not change, and +`--no-smoke-test` and an already-running service are unchanged. + +What is detected, and what is not: + +- Detected (the summary says the smoke test failed): a transport error or timeout, + an HTTP status other than 200, a parse error, and a chunked or `Content-Length` + stream cut short. +- Not detected (no failure is reported and the status reads `ready`): an engine + error sent as a `data:` payload inside a `200` stream; a `200` answer with no + `data:` line at all (HTML, plain JSON, an empty body); a close-delimited stream + cut after a token; and a complete stream with no `content` token, which is + deliberately not a failure (a reasoning model can spend its whole small budget + first, as the readiness probe also allows for). The metric rows read `n/a` when + no content token arrived first; tokens that did arrive (the close-delimited cut, + or an in-band `data:` error after content) are measured. +- Known limitation: the reason shown is whatever the provider call reported, and + some of its errors are not the engine's, for example the `/v1/models` re-list + before the request ("no ready managed service"), an unreadable services-directory + entry, a rejected endpoint URL, and the audit event written after the stream is + consumed. Each turns into a failed smoke test, which is why the note says the + smoke test could not complete and only that the server may not be able to serve + requests. +- Known limitation: the smoke request is routed by model name, not to the launched + service. The newest ready service serving the model is used, so normally the one + just launched. If the new service fails the 2 s `/v1/models` listing probe (for + example the engine was killed just as it became ready) it is filtered out: an + older ready service with the same model then answers, and the summary shows + `ready` with that service's metrics and no note, a false pass on a dead launch. + With no older service the reason shown is rocm's "no ready managed service", not + the engine's. + +Focused unit coverage: + +```bash +cargo test -p rocm --bin rocm serve_smoke_test +cargo test -p rocm --bin rocm a_failed_smoke_test_is_not_presented_as_ready_and_says_why +cargo test -p rocm --bin rocm a_passing_or_skipped_smoke_test_adds_no_failure_note +cargo test -p rocm --bin rocm an_already_running_summary_ignores_a_smoke_failure +cargo test -p rocm --bin rocm failure_reason_ +cargo test -p rocm --bin rocm the_commands_the_failure_note_names_exist_and_parse +``` + +The `serve_smoke_test` tests in [providers.rs](../apps/rocm/src/providers.rs) +run the real smoke test against a local mock engine over a socket (an engine that +dies mid-answer or inside a chunk, a 500, 401 and 404, a hostile status line, a +complete stream with no content, a close-delimited stream cut after a token). Only +the mid-stream-death test goes on to render the summary from the real outcome; the +others assert on the outcome the smoke test returns. + +- `@id:serve-smoke-failure-summary` (serve-60) — a launch that becomes ready + whose engine fails its first chat request is shown as a failed smoke test, with + the engine's own reason and the full `rocm logs --service ` command, and + the engine received exactly one chat request. It carries `@requires-no-gpu`, + `@requires-os:linux` and `@requires-oom-fault-injection`: it uses the same + test-only hook as `@id:serve-oom-launch-memory-guidance` (see [Serve OOM Guidance](#serve-oom-guidance)) in its + `ready` mode, a ready record pointing at a mock engine whose chat requests all + fail with HTTP 500, so it has that scenario's gates: against a prebuilt + `ROCM_CLI_BINARY` the hook is compiled out and the harness reports it as a skip, + unless it is selected with `-n` or `--tags`, which bypasses that skip and makes + it fail for the wrong reason. Some of its steps are not checks of the fix, so say + plainly what they prove: + - The exit code 0 is asserted in the `When` step (`wait_for_exit`), not in a + `Then` step. + - "the engine received exactly one chat request" passes with the fix reverted + (the smoke request was already sent); it shows the request was not retried. + - "the engine's own reason" matches rocm's wording of the status line around the + engine's HTTP status, so it pins that phrasing as well. + - "the launch's service record is left as ready" passes with the fix reverted; + it is the only assertion behind the "ROCm CLI does not stop the server" + promise in the README and `rocm serve --help`. + - "no background automation daemon was left behind" is hygiene only: it records + that no daemon was started but does not discriminate the + `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs`, because no watcher is + enabled in the isolated config and the helper starts only when one is. + +```bash +cargo xtask e2e -- -n "serve-60" +``` + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 99e9ede71..d871ab21d 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -145,7 +145,7 @@ Scenarios carry stable-id and capability tags: | `@requires-case-sensitive-fs` | Premise is two files whose names differ only in letter case. Resolves to **skip** where the scenarios' temp root folds case. The capability probe's `case_sensitive_fs` writes two such names in its own temp dir, which follows `$TMPDIR` like every scenario's isolated root, and checks they stayed two files. `@requires-os:linux` cannot express this: `$TMPDIR` on Linux, WSL2 included, can sit on a casefolded directory or a mounted Windows drive. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | -| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch that runs out of GPU memory so the OOM guidance can be verified without a GPU. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | +| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch so `rocm serve`'s summary can be verified without a GPU: one that runs out of GPU memory (the OOM guidance), or, in its `ready` mode, one that becomes ready and then fails its smoke test. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios, the native-Windows vLLM refusal scenarios serve-33/34/35, and the side-by-side runtime scenarios runtime-lifecycle-19/20/21. | diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a67d286c6..096bd0825 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -17,7 +17,7 @@ Feature: Model serving # This discriminates nothing yet: the scenario is an expected failure on every # host while its xfail row stands, and the vLLM run stops before printing a # plan, so the step panics there whatever the lemonade half did. It becomes - # regression coverage when that row is narrowed. serve-55 and the unit tests in + # regression coverage when that row is narrowed. serve-60 and the unit tests in # serve_cmd.rs are what hold the per-engine refs in the meantime. @id:serve-short-name-consistent-across-engines Scenario: serve-02 - Every engine expands a short name rather than serving it @@ -963,3 +963,20 @@ Feature: Model serving Given a managed runtime is active When the user serves a model that requires an API key Then the server's API key is kept where only that user can read it + + # A launch that became ready but whose first chat request fails must not be + # shown as a healthy deployment (#523). Same test-only hook as + # @id:serve-oom-launch-memory-guidance, in its + # `ready` mode, pointed at a mock engine that fails every chat request. The + # unit and provider tests prove the rendering and the failure classes; this + # proves `serve()` carries the smoke test's failure into the summary. + @id:serve-smoke-failure-summary @requires-no-gpu @requires-os:linux @requires-oom-fault-injection + Scenario: serve-60 - A launch whose first chat request fails is not presented as ready + Given a managed vLLM launch that becomes ready but whose engine fails its first chat request + When the user opens the interactive serve summary against that engine + Then the deployment summary says the smoke test failed and does not call the service ready + And the deployment summary gives the engine's own reason + And the engine received exactly one chat request + And the deployment summary names the logs command for that service + And the launch's service record is left as ready + And no background automation daemon was left behind diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index d63549166..d8857f43c 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -148,9 +148,9 @@ pub struct HostCapability { /// "mock" (no AMD GPU), else the family/target (e.g. "mi300x", "strix-halo"). pub platform_slug: String, /// Whether the `rocm` binary under test carries the test-only - /// `e2e-oom-fault-injection` hook, so `@requires-oom-fault-injection` - /// scenarios can simulate a GPU-less OOM launch. It is compiled out of the - /// shipping binary, and the harness cannot probe for it (no product command + /// `e2e-oom-fault-injection` hook, so the scenarios tagged + /// `@requires-oom-fault-injection` can simulate a GPU-less launch. It is + /// compiled out of the shipping binary, and the harness cannot probe for it (no product command /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with /// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted @@ -1442,9 +1442,9 @@ Local model engines /// `expectation.rs`'s `requires_oom_fault_injection_runs_only_when_the_hook_is_built_in` /// builds `HostCapability { oom_fault_injection: .. }` by hand and so never /// reaches this comparison; without this test, editing the literal on one - /// side of the producer/consumer pair would make serve-41 a permanent skip - /// with nothing red. The `"1"` asserted here is the same literal - /// `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned + /// side of the producer/consumer pair would make the scenarios tagged + /// `@requires-oom-fault-injection` a permanent skip with nothing red. The + /// `"1"` asserted here is the same literal `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned /// by `release_build_resolves_and_exports_both_cli_binaries` on that side. #[test] fn only_the_exact_value_the_producer_writes_arms_the_hook() { diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6..6771d09dc 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -116,7 +116,8 @@ pub struct ScenarioDecl { pub requires_wsl: bool, /// `@requires-oom-fault-injection`: the scenario drives the test-only /// `e2e-oom-fault-injection` hook (armed by `ROCM_E2E_SIMULATE_OOM_LAUNCH`) - /// to fabricate a GPU-less OOM launch, so it can only run against a binary + /// to fabricate a GPU-less launch (one that OOMs, or one that becomes ready + /// and then fails its smoke test), so it can only run against a binary /// compiled with that feature. `xtask e2e` builds it in when it builds the /// binary itself (the GitHub-hosted mock lane), but the self-hosted lanes /// test a prebuilt shipping release binary that has the hook compiled out — diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index e419365c3..990bb6b18 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -52,9 +52,10 @@ const UNRELATED_LIVE_MODEL: &str = "e2e/unrelated-live-model"; /// [`REUSE_REFUSAL_MODEL`] picked it, so this aliases that constant rather than /// repeating the literal. const LEMONADE_SERVE_TARGET: &str = REUSE_REFUSAL_MODEL; -/// Model id for the OOM-launch scenario, kept distinct from every other e2e -/// serve model so its service records can never collide with another -/// scenario's. +/// Model id for the fabricated-launch scenarios (`@id:serve-oom-launch-memory-guidance` +/// and `@id:serve-smoke-failure-summary`), kept +/// distinct from every other e2e serve model so their service records can never +/// collide with another scenario's. const OOM_LAUNCH_MODEL: &str = "e2e/oom-launch-model"; /// How long to wait for a freshly served model's endpoint to become ready. /// @@ -3220,6 +3221,173 @@ async fn assert_oom_launch_quotes_the_real_line(world: &mut E2eWorld) { ); } +/// A managed vLLM launch that became ready, whose engine (a mock) answers +/// `/v1/models` but fails every chat request with HTTP 500. The launch is +/// fabricated by the `ready` mode of the same test-only hook that +/// `@id:serve-oom-launch-memory-guidance` arms; the +/// mock's port is passed to `rocm serve` by the `When` below, which is how the +/// fabricated record points the post-launch smoke test at it. +#[given("a managed vLLM launch that becomes ready but whose engine fails its first chat request")] +async fn plant_ready_launch_with_failing_engine(world: &mut E2eWorld) { + // An empty chat script means this engine always fails chat: `/v1/models` + // answers, and every chat request is "past the end of the script" and gets + // HTTP 500. + let mock = MockServer::start_with_chat_script(OOM_LAUNCH_MODEL, Vec::new()).await; + world.mock = Some(mock); + // Consumed by `e2e_simulate_oom_launch` in + // `apps/rocm/src/service_launch/readiness.rs`; the name is spelled out on both + // sides because the binary's crate cannot depend on this harness. + world + .command_env + .push(("ROCM_E2E_SIMULATE_OOM_LAUNCH", "ready".into())); +} + +#[when("the user opens the interactive serve summary against that engine")] +async fn open_summary_against_failing_engine(world: &mut E2eWorld) { + let port = world + .mock + .as_ref() + .expect("no mock engine was started") + .port() + .to_string(); + let mut session = TuiSession::spawn( + world, + &[ + "serve", + OOM_LAUNCH_MODEL, + "--engine", + "vllm", + "--env-id", + "e2e-smoke-failure", + "--port", + &port, + ], + ) + .unwrap_or_else(|error| panic!("failed to open interactive serve summary: {error}")); + // `wait_for_exit` requires exit code 0: a failed smoke test must not fail + // the launch. + session + .wait_for_exit(interactive_summary_timeout()) + .await + .unwrap_or_else(|error| panic!("interactive serve summary failed: {error}")); + world.tui = Some(session); +} + +#[then("the deployment summary says the smoke test failed and does not call the service ready")] +async fn assert_summary_reports_failed_smoke_test(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains("Deploymentsummary(smoketestfailed)"), + "the heading must say the smoke test failed:\n{screen}" + ); + assert!( + flattened.contains("started,smoketestfailed"), + "the status row must not read `ready`:\n{screen}" + ); + assert!( + !flattened.contains("statusready"), + "a failed smoke test must not leave `status ready` on screen:\n{screen}" + ); +} + +/// Matches rocm's own wording of the status line (`the smoke test could not +/// complete`, `local provider returned ...`) around the engine's HTTP status, so +/// it pins that phrasing as well as the engine's reason. +#[then("the deployment summary gives the engine's own reason")] +async fn assert_summary_gives_engine_reason(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + // The exact reason the provider reports for the mock's answer, not just a + // `500` that the ephemeral port or a timestamp-based service id could also + // contain, and not the reason of some other failed outcome (for example "no + // ready managed service"). + assert!( + flattened.contains("thesmoketestcouldnotcomplete") + && flattened.contains("localproviderreturnedHTTP/1.1500InternalServerError"), + "the note must carry the engine's HTTP 500 as the reason:\n{screen}" + ); +} + +/// Passes with the fix reverted: the smoke request was sent before the fix too. +/// It proves only that the summary's request reached the mock once and was not +/// retried, so the failure the summary shows is that one request's. +#[then("the engine received exactly one chat request")] +async fn assert_engine_received_one_chat_request(world: &mut E2eWorld) { + let received = world + .mock + .as_ref() + .expect("no mock engine was started") + .chat_requests() + .len(); + assert_eq!( + received, 1, + "the smoke test must send exactly one chat request to the engine" + ); +} + +#[then("the deployment summary names the logs command for that service")] +async fn assert_summary_names_logs_command(world: &mut E2eWorld) { + let service_id = launched_service_record(world)["service_id"] + .as_str() + .expect("the service record has no service_id") + .to_owned(); + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains(&format!("check`rocmlogs--service{service_id}`")), + "the note must name `rocm logs --service {service_id}` in full:\n{screen}" + ); +} + +/// The only assertion behind the "ROCm CLI does not stop the server" promise +/// (README and `rocm serve --help`): a failed smoke test leaves the service +/// record `ready`. It passes with the fix reverted, so it does not check the +/// summary change; the steps above do. The exit code 0 is asserted in the `When` +/// step (`wait_for_exit`). +#[then("the launch's service record is left as ready")] +async fn assert_service_record_unchanged(world: &mut E2eWorld) { + assert_eq!( + launched_service_record(world)["status"], + "ready", + "a failed smoke test must not change the service record" + ); +} + +/// The one service record the fabricated launch wrote. +fn launched_service_record(world: &E2eWorld) -> serde_json::Value { + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let mut records: Vec = std::fs::read_dir(&services) + .expect("the launch left no services directory") + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .map(|entry| { + serde_json::from_slice(&std::fs::read(entry.path()).expect("record is readable")) + .expect("record is JSON") + }) + .collect(); + assert_eq!(records.len(), 1, "expected exactly the launched service"); + records.remove(0) +} + /// Collapse a rendered PTY screen to its non-whitespace characters, so an /// assertion survives the terminal soft-wrapping a long line across grid rows /// (which inserts row breaks mid-phrase / mid-token). Only meaningful for @@ -3823,20 +3991,25 @@ async fn serve_wait_refused(world: &mut E2eWorld) { ); } -/// The discriminating assertion for the fabricated-launch daemon suppression. +/// Hook hygiene, shared by the `@id:serve-oom-launch-memory-guidance` and +/// `@id:serve-smoke-failure-summary` scenarios: it records that no daemon was +/// started, and it does NOT discriminate the `if !e2e_simulate_oom_launch()` +/// guard in `serve_cmd.rs`. `ensure_background_helper_running_with` returns early +/// unless a watcher is enabled, and the isolated config these scenarios run in +/// enables none, so removing the guard leaves both scenarios green. /// -/// `simulate_oom_managed_launch` writes a `starting` record for a process that -/// never existed, so `ensure_background_helper_running_quiet` would leave a +/// `simulate_oom_managed_launch` writes a record (`starting`, or `ready` in its +/// `ready` mode) for a process that never existed, so +/// `ensure_background_helper_running_quiet` would leave a /// `rocm daemon` supervising a pid that will never appear — and nothing in this /// scenario's teardown stops it. Observed in the wild as daemons outliving their /// e2e checkout by days. /// -/// Without this step the suppression was untested: deleting the -/// `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs` left serve-41 green, -/// because every other assertion here is about the summary text. Both artefacts -/// the autostart path can leave are checked, since the daemon publishes its -/// runtime-state only after it is up — a spawn caught in the window before that -/// shows up as a claim and nothing else. +/// The guard would only matter in a config with a watcher enabled, which no +/// scenario here sets up; this step would catch a daemon in that case. Both +/// artefacts the autostart path can leave are checked, since the daemon +/// publishes its runtime-state only after it is up — a spawn caught in the +/// window before that shows up as a claim and nothing else. #[then("no background automation daemon was left behind")] async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let root = world @@ -3854,7 +4027,7 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { assert_ne!( state.get("running").and_then(serde_json::Value::as_bool), Some(true), - "a fabricated OOM launch must not start the automation daemon, but \ + "a fabricated launch must not start the automation daemon, but \ {} says one is running:\n{text}", state_path.display() ); @@ -3863,7 +4036,7 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let claim_path = root.join("autostart.claim"); assert!( !claim_path.is_file(), - "a fabricated OOM launch must not even claim a daemon autostart, but {} exists: {:?}", + "a fabricated launch must not even claim a daemon autostart, but {} exists: {:?}", claim_path.display(), std::fs::read_to_string(&claim_path).ok() ); diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 1d8cd0e53..3b376ed73 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -106,9 +106,10 @@ pub fn run(args: &[String]) -> Result<()> { if binaries.build_release { let status = Command::new(&cargo) // `--features rocm/e2e-oom-fault-injection` compiles in the test-only - // `rocm serve` OOM-launch fault injection used by the positive - // `serve-oom-launch-memory-guidance` scenario. It is scoped to the - // `rocm` package (rocmd has no such feature) and to this e2e build + // `rocm serve` launch fault injection (an OOM launch, or a ready + // launch) used by the scenarios tagged `@requires-oom-fault-injection`. + // It is scoped to the `rocm` package (rocmd has no such feature) and + // to this e2e build // only — the release build never enables it, so shipped binaries // carry no fault-injection hook. It joins `rocm/e2e-test-hooks` // (the general e2e hook surface) in a single space-separated