From e5bcd69f4c0f33f86d8de016040207f354074d22 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Wed, 11 Nov 2026 23:17:49 +0000 Subject: [PATCH 1/4] fix(serve): say so in the summary when the post-launch smoke test fails (ROCMAI-703) After `rocm serve`, the deployment summary read `status: ready` with n/a metrics even when the first test request had failed: run_smoke_test discarded the error and returned empty metrics, so a server that could not answer looked the same as one that was not measured. run_smoke_test now returns the failure reason. When the test fails, the summary is headed "(smoke test failed)", the status reads "started, smoke test failed", and a note gives a one-line reason and points at `rocm logs`. The launch stays best-effort: the server keeps running and the exit code is unchanged. Fixes #523 Signed-off-by: Tomas Saaristola --- README.md | 6 +- apps/rocm/src/cli.rs | 3 + apps/rocm/src/providers.rs | 56 ++++++++++- apps/rocm/src/serve_cmd.rs | 22 +++-- apps/rocm/src/serve_summary.rs | 176 +++++++++++++++++++++++++++++++-- 5 files changed, 242 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index bb85d199b9..e2a152f68f 100644 --- a/README.md +++ b/README.md @@ -1262,7 +1262,11 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). Control +quick smoke test (time to first token and approximate tokens per second). If the +smoke test request fails, the summary is headed `(smoke test failed)`, shows the +status as `started, smoke test failed` instead of `ready`, and a note gives the +reason and points at `rocm logs --service `; the launch itself is unchanged +and the server keeps running. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 8b4f2b79f0..226128c60a 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -560,6 +560,9 @@ rocm serve qwen --verbose --device gpu_required")] #[arg(long, conflicts_with = "managed")] verbose: bool, /// Skip the post-startup inference smoke test (time-to-first-token, tokens/sec). + /// + /// When the smoke test runs and its request fails, the summary says so + /// and does not report the service as ready; the server keeps running. #[arg(long)] no_smoke_test: bool, /// Allow binding to a non-local address. diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index d35d738698..9bb0cf3966 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1750,6 +1750,16 @@ mod tests { /// leaves behind. fn ready_service_that_dies_mid_chunked_stream( name: &str, + ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { + ready_service_streaming_one_chunk_then_closing( + name, + "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n", + ) + } + + fn ready_service_streaming_one_chunk_then_closing( + name: &str, + chunk: &'static str, ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { let (root, paths) = temp_app_paths(name); paths.ensure()?; @@ -1763,7 +1773,6 @@ mod tests { stream, "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" )?; - let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; stream.flush()?; // Returning drops the stream: the client sees EOF on a chunk boundary. @@ -1902,15 +1911,20 @@ mod tests { let (outcome, served) = client_then_server(&receiver, server); fs::remove_dir_all(root).ok(); - let metrics = outcome.map_err(|_| { + let smoke = outcome.map_err(|_| { anyhow::anyhow!( "the serve smoke test never returned after the engine closed the connection" ) })?; - // Empty metrics are also what a smoke test that never reached the + // A failed outcome is also what a smoke test that never reached the // engine returns, so this is what makes the assertion below mean // "the engine died mid-answer" rather than "nothing happened". served.transpose()?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke.clone() else { + anyhow::bail!("a stream that died mid-answer was reported as measured: {smoke:?}"); + }; + assert!(!reason.is_empty() && !reason.contains('\n'), "{reason:?}"); + let (metrics, smoke_failure) = smoke.into_parts(); assert_eq!(metrics, crate::serve_summary::SmokeMetrics::default()); let rendered = crate::serve_summary::render_summary(&crate::serve_summary::DeploymentSummary { @@ -1923,11 +1937,47 @@ mod tests { status: "ready".to_owned(), already_running: false, metrics, + smoke_failure, api_key: None, notes: Vec::new(), }); + assert!( + rendered.starts_with("Deployment summary (smoke test failed)\n"), + "{rendered}" + ); + assert!( + rendered.contains("started, smoke test failed"), + "{rendered}" + ); + assert!(!rendered.contains(" ready\n"), "{rendered}"); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains(&reason), "{rendered}"); + assert!(rendered.contains("`rocm logs --service svc-ready`")); + Ok(()) + } + + /// A 200 stream that ends without ever carrying content (an engine that + /// accepts the request and answers with nothing) is a failed smoke test, not + /// a ready deployment with empty metrics. + #[test] + fn serve_smoke_test_fails_when_the_stream_carries_no_content() -> Result<()> { + let (root, paths, server) = ready_service_streaming_one_chunk_then_closing( + "serve-smoke-no-content", + "data: [DONE]\n\n", + )?; + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); + }); + let (outcome, served) = client_then_server(&receiver, server); + fs::remove_dir_all(root).ok(); + let smoke = outcome.map_err(|_| anyhow::anyhow!("the smoke test never returned"))?; + served.transpose()?; + assert!( + matches!(&smoke, crate::serve_summary::SmokeOutcome::Failed(reason) if !reason.is_empty()), + "an empty stream was not reported as a failure: {smoke:?}" + ); Ok(()) } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index cc1da73430..3d49864ef6 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1236,16 +1236,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Best-effort inference smoke test, on by default (opt out with // `--no-smoke-test`). Only meaningful for a freshly-ready server we // just launched; skipped when metrics could not be shown anyway. - let metrics = if !no_smoke_test && !report.already_running && report.status == "ready" { - spinner.set_label("Running smoke test…"); - // The local provider resolves the endpoint key from the per-service - // owner-only key file by service id, so the smoke test authenticates - // against a protected public endpoint without threading the secret - // through here. - serve_summary::run_smoke_test(&paths, resolve.served_model_name()) - } else { - serve_summary::SmokeMetrics::default() - }; + let (metrics, smoke_failure) = + if !no_smoke_test && !report.already_running && report.status == "ready" { + spinner.set_label("Running smoke test…"); + // The local provider resolves the endpoint key from the per-service + // owner-only key file by service id, so the smoke test authenticates + // against a protected public endpoint without threading the secret + // through here. + serve_summary::run_smoke_test(&paths, resolve.served_model_name()).into_parts() + } else { + (serve_summary::SmokeMetrics::default(), None) + }; spinner.clear(); let mut notes = collect_serve_notes( @@ -1314,6 +1315,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { status: report.status.clone(), already_running: report.already_running, metrics, + smoke_failure, api_key: launched_key, notes, }; diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index cd0ab34bc6..f36162d4f0 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -62,6 +62,10 @@ pub(crate) struct DeploymentSummary { /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, pub metrics: SmokeMetrics, + /// Why the post-launch smoke test failed, as one sanitised line. `None` when + /// the test passed or was not run; `Some` keeps the summary from presenting + /// the service as ready. + pub smoke_failure: Option, /// The endpoint API key for a freshly launched *public* server, shown once as /// secure client configuration. `None` for loopback binds (no auth) and for an /// already-running service (whose key was delivered at its own launch). @@ -70,6 +74,46 @@ pub(crate) struct DeploymentSummary { pub notes: Vec, } +/// Result of the post-launch smoke test. +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum SmokeOutcome { + /// The server streamed a first token; the metrics are what was measured. + Measured(SmokeMetrics), + /// The request errored or produced no content; carries a one-line reason. + Failed(String), +} + +impl SmokeOutcome { + /// Splits the outcome into the summary's `metrics` and `smoke_failure` fields. + pub(crate) fn into_parts(self) -> (SmokeMetrics, Option) { + match self { + Self::Measured(metrics) => (metrics, None), + Self::Failed(reason) => (SmokeMetrics::default(), Some(reason)), + } + } +} + +/// Longest failure reason kept for display. +const SMOKE_FAILURE_MAX_CHARS: usize = 200; + +/// Reduces an error message (which may embed an engine's HTTP error body +/// verbatim) to one printable line: control characters and runs of whitespace +/// become a single space, and the result is length-capped. +fn one_line_reason(raw: &str) -> String { + let collapsed = raw + .split(|ch: char| ch.is_whitespace() || ch.is_control()) + .filter(|part| !part.is_empty()) + .collect::>() + .join(" "); + if collapsed.chars().count() > SMOKE_FAILURE_MAX_CHARS { + let mut cut: String = collapsed.chars().take(SMOKE_FAILURE_MAX_CHARS).collect(); + cut.push_str("..."); + cut + } else { + collapsed + } +} + /// Render `Some(ttft)` as a human duration, or `n/a` when the probe did not /// produce a first token. pub(crate) fn format_ttft(ttft: Option) -> String { @@ -102,8 +146,11 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { // note must make that visibly different from a healthy deployment so the // summary is never mistaken for success. let ready = summary.status == "ready"; + let smoke_failed = summary.smoke_failure.is_some() && !summary.already_running; let heading = if summary.already_running { "Deployment summary (already running)" + } else if smoke_failed { + "Deployment summary (smoke test failed)" } else if ready { "Deployment summary" } else { @@ -132,7 +179,14 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { .collect(); let rows: Vec<(&str, String)> = [ vec![ - ("status", summary.status.clone()), + ( + "status", + if smoke_failed { + "started, smoke test failed".to_owned() + } else { + summary.status.clone() + }, + ), ("engine", summary.engine.clone()), ], model_rows, @@ -189,6 +243,15 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { summary.service_id ); } + if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { + // The server answered its health check, but the first request failed, so + // it is not known to serve. Say why the metrics rows are empty. + let _ = writeln!( + out, + " note: the smoke test request failed, so no metrics were measured and the server may not be able to serve requests: {reason} - check `rocm logs --service {id}` and `rocm services list`", + id = summary.service_id + ); + } for note in &summary.notes { let _ = writeln!(out, " note: {note}"); } @@ -281,9 +344,10 @@ pub(crate) fn oom_memory_note( /// Run a single small chat completion against the just-started local server and /// measure time-to-first-token and generation throughput. Best-effort: any error -/// (server not OpenAI-compatible, refused, timed out) yields empty metrics rather -/// than failing the launch. -pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics { +/// (server not OpenAI-compatible, refused, timed out, died mid-answer) or a +/// response with no content yields [`SmokeOutcome::Failed`] with the reason, +/// rather than failing the launch. +pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome { let request = ChatRequest { model: Some(api_model.to_owned()), messages: vec![ChatMessage { @@ -316,11 +380,19 @@ pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeMetrics }, ); - if result.is_err() { - return SmokeMetrics::default(); + if let Err(err) = result { + return SmokeOutcome::Failed(one_line_reason(&format!("{err:#}"))); + } + if first_token.is_none() { + return SmokeOutcome::Failed("the server returned no content".to_owned()); } - compute_metrics(start, first_token, last_token, token_chunks) + SmokeOutcome::Measured(compute_metrics( + start, + first_token, + last_token, + token_chunks, + )) } /// Pure metric math, split out so it can be unit-tested without a live server. @@ -466,6 +538,7 @@ mod tests { ttft: Some(Duration::from_millis(180)), gen_tps: Some(42.15), }, + smoke_failure: None, api_key: None, notes: Vec::new(), } @@ -1160,4 +1233,93 @@ mod tests { "an OOM line beyond the shared tail budget must not surface the note: {notes:?}" ); } + + fn failed_summary(reason: &str) -> DeploymentSummary { + let mut summary = base_summary(); + summary.metrics = SmokeMetrics::default(); + summary.smoke_failure = Some(reason.to_owned()); + summary + } + + #[test] + fn a_failed_smoke_test_is_not_presented_as_ready_and_says_why() { + let rendered = render_summary(&failed_summary("connection closed mid-response")); + assert!( + rendered.starts_with("Deployment summary (smoke test failed)\n"), + "{rendered}" + ); + assert!( + rendered.contains("status started, smoke test failed"), + "{rendered}" + ); + assert!( + !rendered.contains("status ready"), + "{rendered}" + ); + assert!(rendered.contains("time to first token n/a"), "{rendered}"); + assert!(rendered.contains("throughput (approx) n/a"), "{rendered}"); + assert!( + rendered.contains("connection closed mid-response"), + "{rendered}" + ); + // The remediation names commands that exist and take these arguments. + assert!( + rendered + .contains("`rocm logs --service vllm-qwen-1720000000` and `rocm services list`"), + "{rendered}" + ); + } + + #[test] + fn a_passing_or_skipped_smoke_test_adds_no_failure_note() { + let passed = render_summary(&base_summary()); + assert!(passed.starts_with("Deployment summary\n"), "{passed}"); + assert!(!passed.contains("smoke test"), "{passed}"); + // `--no-smoke-test`: default metrics, no failure recorded. + let mut skipped = base_summary(); + skipped.metrics = SmokeMetrics::default(); + let skipped = render_summary(&skipped); + assert!(skipped.starts_with("Deployment summary\n"), "{skipped}"); + assert!(!skipped.contains("smoke test"), "{skipped}"); + assert!(skipped.contains("status ready"), "{skipped}"); + } + + #[test] + fn an_already_running_summary_ignores_a_smoke_failure() { + let mut summary = failed_summary("boom"); + summary.already_running = true; + let rendered = render_summary(&summary); + assert!( + rendered.starts_with("Deployment summary (already running)\n"), + "{rendered}" + ); + assert!(!rendered.contains("smoke test"), "{rendered}"); + } + + #[test] + fn smoke_outcome_splits_into_summary_fields() { + let metrics = SmokeMetrics { + ttft: Some(Duration::from_millis(5)), + gen_tps: None, + }; + assert_eq!( + SmokeOutcome::Measured(metrics.clone()).into_parts(), + (metrics, None) + ); + assert_eq!( + SmokeOutcome::Failed("x".to_owned()).into_parts(), + (SmokeMetrics::default(), Some("x".to_owned())) + ); + } + + #[test] + fn failure_reason_is_one_line_without_control_characters_and_capped() { + let reason = one_line_reason("HTTP 500\r\n{\"error\":\t\"boom\"}\u{1b}[31m red\u{7}\n\n"); + assert!(!reason.chars().any(char::is_control), "{reason:?}"); + assert_eq!(reason, "HTTP 500 {\"error\": \"boom\"} [31m red"); + let long = one_line_reason(&"x".repeat(5000)); + assert_eq!(long.chars().count(), SMOKE_FAILURE_MAX_CHARS + 3); + assert!(long.ends_with("...")); + assert_eq!(one_line_reason("short"), "short"); + } } From 3aa67a8c164b1f35f5a39a5c46d026d8f56ad631 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 09:50:33 +0000 Subject: [PATCH 2/4] fix(serve): assert the smoke-failure summary end to end and without echoing the api key (ROCMAI-703) The first commit's tests echoed the rendered summary, which carries the endpoint api key, into assert messages, and CodeQL rust/cleartext-logging flagged them. The asserts now carry constant messages. Also in this change: - the failure reason is cleared of terminal control and format characters, and text after a cut-off escape sequence is kept; - a clean stream with no content token (a reasoning model) is no longer treated as a failure; - serve-55, a second `ready` mode of the existing e2e fault-injection hook (compiled out of release builds), asserts the heading, status row, engine reason, one chat request and the logs command over a pty with exit 0; - the README and test docs state what is and is not detected. Signed-off-by: Tomas Saaristola --- README.md | 11 +- apps/rocm/Cargo.toml | 6 +- apps/rocm/src/cli.rs | 4 +- apps/rocm/src/providers.rs | 270 +++++++++++++----- apps/rocm/src/serve_cmd.rs | 16 +- apps/rocm/src/serve_summary.rs | 184 ++++++++---- apps/rocm/src/service_launch.rs | 4 +- apps/rocm/src/service_launch/readiness.rs | 55 +++- docs/architecture.md | 2 +- docs/manual-testing.md | 37 +++ docs/testing.md | 81 +++++- tests/e2e-cucumber/README.md | 2 +- .../features/model_serving.feature | 18 +- tests/e2e-cucumber/src/capability.rs | 6 +- tests/e2e-cucumber/src/expectation.rs | 3 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 175 +++++++++++- xtask/src/e2e.rs | 5 +- 17 files changed, 696 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index e2a152f68f..f8927a3f3f 100644 --- a/README.md +++ b/README.md @@ -1262,11 +1262,12 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens per second). If the -smoke test request fails, the summary is headed `(smoke test failed)`, shows the -status as `started, smoke test failed` instead of `ready`, and a note gives the -reason and points at `rocm logs --service `; the launch itself is unchanged -and the server keeps running. Control +quick smoke test (time to first token and approximate tokens per second). On an +interactive terminal, if the smoke test cannot complete, the summary is headed +`(smoke test failed)`, shows the status as `started, smoke test failed` instead +of `ready`, and a note gives the reason and points at +`rocm logs --service `. ROCm CLI does not stop the server, and the command +still exits 0. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index c253f67744..4b72d65640 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -19,8 +19,10 @@ e2e-test-hooks = [ ] # Test-only fault injection for the e2e suite. Enabled ONLY by the `xtask e2e` # mock-lane build (never by the release build), it compiles in a `rocm serve` -# hook that simulates a managed vLLM launch which OOM'd and never became ready, -# so the positive OOM-guidance summary can be verified on a GPU-less CI host. +# hook that simulates a managed vLLM launch, so the `rocm serve` summary can be +# verified on a GPU-less CI host: one that OOM'd and never became ready (the +# OOM guidance), or, in the hook's `ready` mode, one that became ready and then +# fails its smoke test. # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in # `service_launch/readiness.rs`. e2e-oom-fault-injection = [] diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 226128c60a..9130f148d2 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -561,8 +561,8 @@ rocm serve qwen --verbose --device gpu_required")] verbose: bool, /// Skip the post-startup inference smoke test (time-to-first-token, tokens/sec). /// - /// When the smoke test runs and its request fails, the summary says so - /// and does not report the service as ready; the server keeps running. + /// When the smoke test runs and cannot complete, the summary says so + /// and does not report the service as ready; ROCm CLI does not stop the server. #[arg(long)] no_smoke_test: bool, /// Allow binding to a non-local address. diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 9bb0cf3966..f3b9fd7e68 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -1673,22 +1673,16 @@ mod tests { paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + write_chunked_sse_head(stream)?; let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; let done = "data: [DONE]\n\n"; write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", done.len(), done)?; stream.flush()?; Ok(()) - }, - ); + }); let mut ready = ManagedServiceRecord::new( &paths, @@ -1751,34 +1745,41 @@ mod tests { fn ready_service_that_dies_mid_chunked_stream( name: &str, ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { - ready_service_streaming_one_chunk_then_closing( - name, - "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n", - ) + ready_service_answering_chat_with(name, answer_then_die_on_a_chunk_boundary) + } + + /// Sends the chunked SSE head and one content chunk, then returns, which + /// drops the stream: the client sees EOF on a chunk boundary. + fn answer_then_die_on_a_chunk_boundary(stream: &mut TcpStream) -> Result<()> { + write_chunked_sse_head(stream)?; + let chunk = "data: {\"choices\":[{\"delta\":{\"content\":\"chunk\"}}]}\n\n"; + write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; + stream.flush()?; + Ok(()) } - fn ready_service_streaming_one_chunk_then_closing( + fn write_chunked_sse_head(stream: &mut TcpStream) -> Result<()> { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + )?; + Ok(()) + } + + /// A ready local service whose engine answers the chat request with + /// `respond` (after answering any `/v1/models` listing itself). + fn ready_service_answering_chat_with( name: &str, - chunk: &'static str, + mut respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, ) -> Result<(PathBuf, AppPaths, thread::JoinHandle>)> { let (root, paths) = temp_app_paths(name); paths.ensure()?; let listener = TcpListener::bind("127.0.0.1:0")?; let port = listener.local_addr()?.port(); - let server = spawn_local_provider_test_server( - listener, - "Qwen/Qwen3.5", - move |stream, _request| { - write!( - stream, - "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n" - )?; - write!(stream, "{:X}\r\n{}\r\n", chunk.len(), chunk)?; - stream.flush()?; - // Returning drops the stream: the client sees EOF on a chunk boundary. - Ok(()) - }, - ); + let server = + spawn_local_provider_test_server(listener, "Qwen/Qwen3.5", move |stream, _request| { + respond(stream) + }); let mut ready = ManagedServiceRecord::new( &paths, "svc-ready", @@ -1897,35 +1898,31 @@ mod tests { } /// `rocm serve`'s post-launch smoke test is the CLI's streaming consumer. An - /// engine that dies mid-answer must leave the deployment summary showing no - /// measurements (`n/a`) instead of hanging the command or reporting - /// throughput for an answer that never finished. + /// engine that dies mid-answer must make the smoke test fail, and the + /// deployment summary built from that outcome must say so (heading, status + /// and reason, metrics `n/a`) instead of hanging the command, presenting the + /// service as ready, or reporting throughput for an answer that never + /// finished. #[test] - fn serve_smoke_test_reports_no_metrics_when_engine_dies_mid_chunked_stream() -> Result<()> { - let (root, paths, server) = - ready_service_that_dies_mid_chunked_stream("serve-smoke-chunked-truncated")?; - let (sender, receiver) = mpsc::channel(); - thread::spawn(move || { - let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); - }); - let (outcome, served) = client_then_server(&receiver, server); - fs::remove_dir_all(root).ok(); - - let smoke = outcome.map_err(|_| { - anyhow::anyhow!( - "the serve smoke test never returned after the engine closed the connection" - ) - })?; - // A failed outcome is also what a smoke test that never reached the - // engine returns, so this is what makes the assertion below mean + fn serve_smoke_test_fails_and_summary_says_so_when_engine_dies_mid_chunked_stream() -> Result<()> + { + // `smoke_outcome_against` also checks the engine side served its + // request: a failed outcome is also what a smoke test that never reached + // the engine returns, so that is what makes the assertion below mean // "the engine died mid-answer" rather than "nothing happened". - served.transpose()?; + let smoke = smoke_outcome_against( + "serve-smoke-chunked-truncated", + answer_then_die_on_a_chunk_boundary, + )?; + // The asserts below do not interpolate the outcome or the rendered + // summary: `DeploymentSummary` carries an API key field, and echoing it + // trips CodeQL's `rust/cleartext-logging`. let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke.clone() else { - anyhow::bail!("a stream that died mid-answer was reported as measured: {smoke:?}"); + anyhow::bail!("a stream that died mid-answer was reported as measured"); }; - assert!(!reason.is_empty() && !reason.contains('\n'), "{reason:?}"); + // The reason is the provider's own truncation message, not just any text. + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); let (metrics, smoke_failure) = smoke.into_parts(); - assert_eq!(metrics, crate::serve_summary::SmokeMetrics::default()); let rendered = crate::serve_summary::render_summary(&crate::serve_summary::DeploymentSummary { engine: "vllm".to_owned(), @@ -1941,31 +1938,23 @@ mod tests { api_key: None, notes: Vec::new(), }); - assert!( - rendered.starts_with("Deployment summary (smoke test failed)\n"), - "{rendered}" - ); - assert!( - rendered.contains("started, smoke test failed"), - "{rendered}" - ); - assert!(!rendered.contains(" ready\n"), "{rendered}"); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("started, smoke test failed")); + assert!(!rendered.contains(" ready\n")); assert!(rendered.contains("time to first token n/a")); assert!(rendered.contains("throughput (approx) n/a")); - assert!(rendered.contains(&reason), "{rendered}"); + assert!(rendered.contains(&reason)); assert!(rendered.contains("`rocm logs --service svc-ready`")); Ok(()) } - /// A 200 stream that ends without ever carrying content (an engine that - /// accepts the request and answers with nothing) is a failed smoke test, not - /// a ready deployment with empty metrics. - #[test] - fn serve_smoke_test_fails_when_the_stream_carries_no_content() -> Result<()> { - let (root, paths, server) = ready_service_streaming_one_chunk_then_closing( - "serve-smoke-no-content", - "data: [DONE]\n\n", - )?; + /// Runs the smoke test against a ready service whose engine answers the chat + /// request with `respond`, and returns the outcome. + fn smoke_outcome_against( + name: &str, + respond: impl FnMut(&mut TcpStream) -> Result<()> + Send + 'static, + ) -> Result { + let (root, paths, server) = ready_service_answering_chat_with(name, respond)?; let (sender, receiver) = mpsc::channel(); thread::spawn(move || { let _ = sender.send(crate::serve_summary::run_smoke_test(&paths, "Qwen/Qwen3.5")); @@ -1974,13 +1963,144 @@ mod tests { fs::remove_dir_all(root).ok(); let smoke = outcome.map_err(|_| anyhow::anyhow!("the smoke test never returned"))?; served.transpose()?; + Ok(smoke) + } + + /// A chat request that completes without error is not a failed smoke test + /// just because no `content` token arrived: a reasoning model can spend its + /// small token budget on `reasoning_content` first, and the readiness probe + /// (`openai_chat_completion_probe`) already treats such a server as serving. + /// The metrics stay empty (`n/a`) and no failure is recorded. + #[test] + fn serve_smoke_test_is_not_a_failure_when_a_complete_stream_carries_no_content() -> Result<()> { + for (name, body) in [ + ( + "serve-smoke-reasoning-only", + "data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"thinking\"}}]}\n\ndata: [DONE]\n\n", + ), + ("serve-smoke-done-only", "data: [DONE]\n\n"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write_chunked_sse_head(stream)?; + write!(stream, "{:X}\r\n{}\r\n0\r\n\r\n", body.len(), body)?; + stream.flush()?; + Ok(()) + })?; + assert!( + smoke + == crate::serve_summary::SmokeOutcome::Measured( + crate::serve_summary::SmokeMetrics::default() + ), + "a complete stream with no content token was not a measured, empty outcome" + ); + let (metrics, smoke_failure) = smoke.into_parts(); + assert!(smoke_failure.is_none()); + assert!(metrics == crate::serve_summary::SmokeMetrics::default()); + } + Ok(()) + } + + /// The reason in the summary is built from the engine's own words (here its + /// HTTP status line), so it must reach the terminal as one clean line: no + /// escape sequence, no control or format character, but still the status. + #[test] + fn serve_smoke_test_failure_reason_carries_no_terminal_control_sequences() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-hostile-status", |stream| { + write!( + stream, + "HTTP/1.1 500 \u{1b}[31mBoom\u{7}\u{202e}x\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP 500 was not reported as a failure"); + }; + assert!(reason.contains("500")); + assert!(reason.contains("Boom")); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format), + "the failure reason still carries a control or format character" + ); + Ok(()) + } + + /// An engine that dies *inside* a chunk leaves an error with a cause chain + /// (the truncation message wrapping the failed read). The summary's reason + /// must keep the cause, not just the outermost message. + #[test] + fn serve_smoke_test_failure_reason_keeps_the_cause_chain() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-dies-inside-chunk", |stream| { + write_chunked_sse_head(stream)?; + // Promise 0x64 bytes, send three, close. + write!(stream, "64\r\nabc")?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an engine that died inside a chunk was not reported as a failure"); + }; + assert!(reason.contains(LOCAL_STREAM_TRUNCATED)); assert!( - matches!(&smoke, crate::serve_summary::SmokeOutcome::Failed(reason) if !reason.is_empty()), - "an empty stream was not reported as a failure: {smoke:?}" + reason.contains("failed to read local provider SSE body"), + "the reason lost the cause behind the truncation message" ); Ok(()) } + /// Any HTTP error status from the engine is a failed smoke test, with the + /// status in the reason: the check is stricter than the readiness probe's + /// "anything below 500", so a 4xx fails here too. + #[test] + fn serve_smoke_test_fails_on_any_non_200_status() -> Result<()> { + for (name, status_line, code) in [ + ("serve-smoke-http-500", "500 Internal Server Error", "500"), + ("serve-smoke-http-401", "401 Unauthorized", "401"), + ("serve-smoke-http-404", "404 Not Found", "404"), + ] { + let smoke = smoke_outcome_against(name, move |stream| { + write!( + stream, + "HTTP/1.1 {status_line}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Failed(reason) = smoke else { + anyhow::bail!("an HTTP error status was not reported as a failure"); + }; + assert!(reason.contains(code), "the reason lost the status code"); + } + Ok(()) + } + + /// A close-delimited stream (no chunking, no `Content-Length`) that ends + /// after some tokens is deliberately `Measured`: it has no framing to cut + /// short, so a clean close is indistinguishable from completion (see + /// `read_close_delimited_sse_body`). Pins `run_smoke_test`'s doc comment. + #[test] + fn serve_smoke_test_measures_a_close_delimited_stream_cut_after_a_token() -> Result<()> { + let smoke = smoke_outcome_against("serve-smoke-close-delimited", |stream| { + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\nConnection: close\r\n\r\n" + )?; + write!( + stream, + "data: {{\"choices\":[{{\"delta\":{{\"content\":\"hi\"}}}}]}}\n\n" + )?; + stream.flush()?; + Ok(()) + })?; + let crate::serve_summary::SmokeOutcome::Measured(metrics) = smoke else { + anyhow::bail!("a close-delimited stream cut after a token was reported as failed"); + }; + assert!(metrics.ttft.is_some()); + Ok(()) + } + #[test] fn remote_openai_stream_chat_uses_live_sse() -> Result<()> { let listener = TcpListener::bind("127.0.0.1:0")?; diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 3d49864ef6..d56eae3824 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1193,20 +1193,20 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }; let mut spinner = spinner.into_inner(); // Not when the launch was fabricated. `simulate_oom_managed_launch` - // writes a `starting` record for a process that never existed, so the - // helper would be left watching a pid that will never appear, with - // nothing in the scenario's teardown to stop it: observed as `rocm - // daemon` processes outliving their e2e checkout by days, still - // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for + // writes a record (`starting`, or `ready` in its `ready` mode) for a + // process that never existed, so the helper would be left watching a pid + // that will never appear, with nothing in the scenario's teardown to stop + // it: observed as `rocm daemon` processes outliving their e2e checkout by + // days, still carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for // supervising a real spawned server, and there is none here. // // That — not ordering — is the whole justification. This suppression // runs *before* the OOM note is computed in the `summary_mode` block // below, so the previous claim here, that the note "is computed before // it would have run", was backwards and could not have excused it. - // serve-40's `no background automation daemon was left behind` is what - // holds the branch: deleting it left the scenario green with a `rocm - // daemon` still running. + // The `no background automation daemon was left behind` step of serve-41 + // and serve-55 is what holds the branch: deleting it left the scenario + // green with a `rocm daemon` still running. // // `e2e_simulate_oom_launch` is a `const fn` returning `false` without // the test-only feature, so this branch folds away in shipped builds. diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index f36162d4f0..9270564add 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -32,7 +32,9 @@ const SMOKE_PROMPT: &str = "Reply with a short one-sentence greeting."; const SMOKE_MAX_TOKENS: u32 = 32; /// Best-effort metrics measured against a freshly-started server. Every field is -/// optional: any probe failure leaves it `None` and the summary renders `n/a`. +/// optional: `None` renders as `n/a`. They are empty when the smoke test was +/// skipped, failed (see [`SmokeOutcome::Failed`]), or completed without a content +/// token (a reasoning model can spend its whole token budget before emitting any). #[derive(Debug, Default, Clone, PartialEq)] pub(crate) struct SmokeMetrics { /// Time from request send to the first streamed content token. @@ -63,8 +65,9 @@ pub(crate) struct DeploymentSummary { pub already_running: bool, pub metrics: SmokeMetrics, /// Why the post-launch smoke test failed, as one sanitised line. `None` when - /// the test passed or was not run; `Some` keeps the summary from presenting - /// the service as ready. + /// the test passed or was not run. When `Some` (and `already_running` is + /// false, which the renderer checks) the summary is not presented as ready; + /// an already-running service never ran a smoke test and ignores it. pub smoke_failure: Option, /// The endpoint API key for a freshly launched *public* server, shown once as /// secure client configuration. `None` for loopback binds (no auth) and for an @@ -77,9 +80,22 @@ pub(crate) struct DeploymentSummary { /// Result of the post-launch smoke test. #[derive(Debug, Clone, PartialEq)] pub(crate) enum SmokeOutcome { - /// The server streamed a first token; the metrics are what was measured. + /// The request completed with no transport error, no HTTP error status and + /// no truncated stream; the metrics are what was measured. They are empty + /// when no content token arrived, which is not a failure: a reasoning model + /// can spend its small token budget before emitting any, as + /// `openai_chat_completion_probe` also allows for readiness. The status check + /// is deliberately stricter than that probe's (any non-200 fails here, where + /// the probe accepts anything below 500). Not detected here, and so + /// `Measured`: an error reported inside a 200 stream as a `data:` payload, a + /// 200 answer with no `data:` line at all (HTML, plain JSON, an empty body), + /// and a close-delimited stream cut after a token. Measured(SmokeMetrics), - /// The request errored or produced no content; carries a one-line reason. + /// The smoke test could not complete: a transport error, an HTTP error + /// status, a parse error, a chunked or `Content-Length` stream cut short, or + /// a failure in the provider plumbing around the request (finding the ready + /// service, writing the audit event after the stream). Carries a one-line + /// reason. It does not by itself prove the engine failed. Failed(String), } @@ -96,13 +112,20 @@ impl SmokeOutcome { /// Longest failure reason kept for display. const SMOKE_FAILURE_MAX_CHARS: usize = 200; -/// Reduces an error message (which may embed an engine's HTTP error body -/// verbatim) to one printable line: control characters and runs of whitespace -/// become a single space, and the result is length-capped. +/// Reduces an error message to one printable line. On the smoke-test path the +/// engine-controlled text in it is the HTTP status line, reason phrase included +/// (the local provider reports `local provider returned {status_line}`, never the +/// response body), so it is untrusted. Terminal escape sequences and any control +/// or Unicode format character (a bidi override, a zero-width mark) are dropped, +/// line breaks and runs of whitespace become a single space, and the result is +/// length-capped. fn one_line_reason(raw: &str) -> String { - let collapsed = raw - .split(|ch: char| ch.is_whitespace() || ch.is_control()) - .filter(|part| !part.is_empty()) + // The escape scanner, not this function, decides where a line break falls: + // turning breaks into spaces first would let a cut-off escape sequence take + // the next real character as an intermediate byte and swallow it. + let collapsed = rocm_core::terminal::rendered_lines(raw) + .join(" ") + .split_whitespace() .collect::>() .join(" "); if collapsed.chars().count() > SMOKE_FAILURE_MAX_CHARS { @@ -244,11 +267,12 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ); } if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { - // The server answered its health check, but the first request failed, so - // it is not known to serve. Say why the metrics rows are empty. + // The server answered its health check, but the smoke test could not + // complete, so it is not known to serve. Say why the metrics rows are + // empty, hedged: the reason may be rocm's own plumbing, not the engine. let _ = writeln!( out, - " note: the smoke test request failed, so no metrics were measured and the server may not be able to serve requests: {reason} - check `rocm logs --service {id}` and `rocm services list`", + " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`", id = summary.service_id ); } @@ -343,10 +367,17 @@ pub(crate) fn oom_memory_note( } /// Run a single small chat completion against the just-started local server and -/// measure time-to-first-token and generation throughput. Best-effort: any error -/// (server not OpenAI-compatible, refused, timed out, died mid-answer) or a -/// response with no content yields [`SmokeOutcome::Failed`] with the reason, -/// rather than failing the launch. +/// measure time-to-first-token and generation throughput. Best-effort: a request +/// that errors (refused, an HTTP error status other than 200, timed out) or +/// whose stream is a chunked or `Content-Length` body cut short yields +/// [`SmokeOutcome::Failed`] with the reason, rather than failing the launch. So +/// does any other error from the provider call, including two that are not the +/// engine's fault: no ready service found when it re-lists `/v1/models`, and the +/// audit-log write after the stream. A request that completes without a content +/// token is `Measured` with empty metrics, not a failure. Also `Measured`, and so +/// not detected: an error `data:` payload inside a 200 stream, a 200 answer with +/// no `data:` line at all, and a close-delimited stream cut after a token (it has +/// no framing to cut short). pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome { let request = ChatRequest { model: Some(api_model.to_owned()), @@ -383,9 +414,6 @@ pub(crate) fn run_smoke_test(paths: &AppPaths, api_model: &str) -> SmokeOutcome if let Err(err) = result { return SmokeOutcome::Failed(one_line_reason(&format!("{err:#}"))); } - if first_token.is_none() { - return SmokeOutcome::Failed("the server returned no content".to_owned()); - } SmokeOutcome::Measured(compute_metrics( start, @@ -1244,44 +1272,29 @@ mod tests { #[test] fn a_failed_smoke_test_is_not_presented_as_ready_and_says_why() { let rendered = render_summary(&failed_summary("connection closed mid-response")); - assert!( - rendered.starts_with("Deployment summary (smoke test failed)\n"), - "{rendered}" - ); - assert!( - rendered.contains("status started, smoke test failed"), - "{rendered}" - ); - assert!( - !rendered.contains("status ready"), - "{rendered}" - ); - assert!(rendered.contains("time to first token n/a"), "{rendered}"); - assert!(rendered.contains("throughput (approx) n/a"), "{rendered}"); - assert!( - rendered.contains("connection closed mid-response"), - "{rendered}" - ); - // The remediation names commands that exist and take these arguments. - assert!( - rendered - .contains("`rocm logs --service vllm-qwen-1720000000` and `rocm services list`"), - "{rendered}" - ); + assert!(rendered.starts_with("Deployment summary (smoke test failed)\n")); + assert!(rendered.contains("status started, smoke test failed")); + assert!(!rendered.contains("status ready")); + assert!(rendered.contains("time to first token n/a")); + assert!(rendered.contains("throughput (approx) n/a")); + assert!(rendered.contains("connection closed mid-response")); + // That the named commands exist and parse is + // `the_commands_the_failure_note_names_exist_and_parse`. + assert!(rendered.contains("check `rocm logs --service vllm-qwen-1720000000`")); } #[test] fn a_passing_or_skipped_smoke_test_adds_no_failure_note() { let passed = render_summary(&base_summary()); - assert!(passed.starts_with("Deployment summary\n"), "{passed}"); - assert!(!passed.contains("smoke test"), "{passed}"); + assert!(passed.starts_with("Deployment summary\n")); + assert!(!passed.contains("smoke test")); // `--no-smoke-test`: default metrics, no failure recorded. let mut skipped = base_summary(); skipped.metrics = SmokeMetrics::default(); let skipped = render_summary(&skipped); - assert!(skipped.starts_with("Deployment summary\n"), "{skipped}"); - assert!(!skipped.contains("smoke test"), "{skipped}"); - assert!(skipped.contains("status ready"), "{skipped}"); + assert!(skipped.starts_with("Deployment summary\n")); + assert!(!skipped.contains("smoke test")); + assert!(skipped.contains("status ready")); } #[test] @@ -1289,11 +1302,8 @@ mod tests { let mut summary = failed_summary("boom"); summary.already_running = true; let rendered = render_summary(&summary); - assert!( - rendered.starts_with("Deployment summary (already running)\n"), - "{rendered}" - ); - assert!(!rendered.contains("smoke test"), "{rendered}"); + assert!(rendered.starts_with("Deployment summary (already running)\n")); + assert!(!rendered.contains("smoke test")); } #[test] @@ -1315,11 +1325,71 @@ mod tests { #[test] fn failure_reason_is_one_line_without_control_characters_and_capped() { let reason = one_line_reason("HTTP 500\r\n{\"error\":\t\"boom\"}\u{1b}[31m red\u{7}\n\n"); - assert!(!reason.chars().any(char::is_control), "{reason:?}"); - assert_eq!(reason, "HTTP 500 {\"error\": \"boom\"} [31m red"); + assert!( + !reason.chars().any(char::is_control), + "a control character survived" + ); + assert_eq!(reason, "HTTP 500 {\"error\": \"boom\"} red"); let long = one_line_reason(&"x".repeat(5000)); assert_eq!(long.chars().count(), SMOKE_FAILURE_MAX_CHARS + 3); assert!(long.ends_with("...")); assert_eq!(one_line_reason("short"), "short"); } + + #[test] + fn failure_reason_keeps_the_text_after_a_cut_off_escape_sequence() { + // A capture cut mid-sequence must not eat the first real character after + // it: the line break has to reach the escape scanner as a break, not as + // a space it would read as part of the sequence. + for (raw, expected) in [ + ("\u{1b}[31\nHTTP 500", "HTTP 500"), + ("\u{1b}\nHTTP 500", "HTTP 500"), + ("\u{1b}[31mHTTP 500", "HTTP 500"), + ("\u{1b}[31m\nHTTP 500", "HTTP 500"), + ("HTTP\r\n500", "HTTP 500"), + ] { + assert!( + one_line_reason(raw) == expected, + "an escape sequence in the engine's message ate or kept the wrong text" + ); + } + } + + #[test] + fn failure_reason_drops_unicode_format_characters() { + // U+202E (right-to-left override) reorders how the rest of the line + // renders; U+200B and U+FEFF are invisible; U+2028 breaks the row. None + // is matched by `char::is_control`. + let reason = one_line_reason("evil\u{202e}txt\u{200b}.\u{feff}exe\u{2028}next"); + assert_eq!(reason, "eviltxt.exe next"); + assert!( + !reason + .chars() + .any(rocm_core::terminal::is_control_or_format) + ); + } + + #[test] + fn the_commands_the_failure_note_names_exist_and_parse() { + use clap::Parser as _; + let rendered = render_summary(&failed_summary("boom")); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + let commands: Vec<&str> = note.split('`').skip(1).step_by(2).collect(); + // Constant messages only: `commands` is derived from `render_summary`, + // whose input type carries an API key field, and echoing that output + // trips CodeQL's `rust/cleartext-logging`. + assert!( + commands == ["rocm logs --service vllm-qwen-1720000000"], + "the failure note must name exactly the log command" + ); + for command in commands { + assert!( + crate::Cli::try_parse_from(command.split_whitespace()).is_ok(), + "a command named in the failure note is not one rocm accepts" + ); + } + } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index ae5c2fdb5e..7d0e491063 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -42,8 +42,8 @@ //! failed launch, and the reuse refusals. //! - `spawn`: the detached engine child, its process and stdio plumbing, and //! `ManagedLaunchReport`. -//! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection -//! seam. +//! - `readiness`: waiting on a service, its liveness, and the fault-injection +//! seam (an OOM launch, or a ready launch). //! - `start`: the background launch path and its plain output. //! - `helper_autostart`: starting the background automation helper. //! - `attach`: the attached run and its log streaming. diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 1e14c85599..799617553f 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -4,12 +4,13 @@ //! Waiting on a launched or already-live managed service: the readiness budget, //! liveness of the tracked process, how a wait ends, and the end-to-end OOM -//! fault-injection seam that fabricates a launch that never became ready. +//! fault-injection seam that fabricates a launch: one that never became ready +//! (it ran out of GPU memory), or, in the `ready` mode, one that became ready. // There is deliberately no `use std::fs`, `use std::thread` or // `use rocm_core::process_is_running` here: this file names them by full path. // Their use sites sit under different `cfg` combinations (the Linux-only zombie -// probe, the OOM fault-injection seam, one Linux-only test), so any single import +// probe, the fault-injection seam, one Linux-only test), so any single import // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. @@ -180,9 +181,22 @@ pub(super) fn record_wait_outcome( /// out-of-memory traceback to the log it owns, and never became ready — so the /// positive OOM-guidance summary path can be verified on a GPU-less CI host, /// where a real launch cannot run. See [`simulate_oom_managed_launch`]. +/// +/// The value `ready` selects the one other mode: the fabricated launch is a +/// managed vLLM launch that *did* become ready (nothing to blame on memory), so a +/// scenario can point its port at a mock engine and watch what the summary does +/// when the post-launch smoke test then fails. Both values open the same seams +/// (the GPU pre-flight waiver and the background-helper suppression). #[cfg(feature = "e2e-oom-fault-injection")] pub(crate) fn e2e_simulate_oom_launch() -> bool { - std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH") + .is_some_and(|value| value == "1" || value == "ready") +} + +/// Whether the armed hook is in its `ready` mode (see [`e2e_simulate_oom_launch`]). +#[cfg(feature = "e2e-oom-fault-injection")] +fn e2e_simulate_ready_launch() -> bool { + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "ready") } /// Release builds carry no fault-injection switch: the seam compiles out @@ -193,12 +207,18 @@ pub(crate) const fn e2e_simulate_oom_launch() -> bool { } /// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a -/// managed vLLM launch that spawned, OOM'd, and never became ready, then return -/// the launch report the summary path would see for it — `already_running: false` -/// (this invocation launched it), `status: "starting"` (it failed to become -/// ready), and a `log_path` whose contents carry a real allocator OOM signature. -/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is -/// spawned. Compiled out of shipped binaries. +/// managed vLLM launch, then return the launch report the summary path would see +/// for it. `already_running` is false either way (this invocation launched it) +/// and no process is spawned. Compiled out of shipped binaries. +/// +/// - Default mode (`ROCM_E2E_SIMULATE_OOM_LAUNCH=1`): the launch OOM'd and never +/// became ready — `status: "starting"` and a `log_path` whose contents carry a +/// real allocator OOM signature. +/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob +/// guidance. +/// - `ready` mode: the launch became ready — `status: "ready"`, the record marked +/// inference-verified, and `log_path: None` (nothing failed to start), so the +/// caller's smoke test runs against whatever answers on `host:port`. #[cfg(feature = "e2e-oom-fault-injection")] #[allow(clippy::too_many_arguments)] pub(super) fn simulate_oom_managed_launch( @@ -231,6 +251,23 @@ pub(super) fn simulate_oom_managed_launch( Some(device_policy_name(device_policy).to_owned()), ); record.gpu_indices = gpu_indices.to_vec(); + if e2e_simulate_ready_launch() { + // The `ready` mode: a launch that came up. The record is what the local + // provider needs to send the smoke test to `host:port`; no log is written + // because nothing here failed to start. + record.status = "ready".to_owned(); + record.inference_verified_at_unix_ms = Some(1); + record.write()?; + return Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url: format!("{}/v1", format_http_base_url(host, port)), + status: "ready".to_owned(), + already_running: false, + child_pid: None, + log_path: None, + manifest_path: Some(record.manifest_path), + }); + } record.status = "starting".to_owned(); record.write()?; // A real allocator OOM signature in the log this invocation owns, so the diff --git a/docs/architecture.md b/docs/architecture.md index 73a775da2a..8d7c17a37c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -120,7 +120,7 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | | [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. | relocation | -| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the OOM fault-injection seam for end-to-end tests. | relocation | +| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. | relocation | | [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | | [`service_launch/test_support.rs`](../apps/rocm/src/service_launch/test_support.rs) | Test fixtures shared by more than one `service_launch/` child. | flat | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c30a23798e..984ef8651d 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1941,3 +1941,40 @@ Expected result: read (N of M tokens). Try a more specific question, or a model with a larger context.` — and `rocm-cli.log` should still carry the provider's own body with `exceed_context_size_error` in it. + +## 16. Serve Smoke Test Failure + +Needs any engine `rocm serve` can start (the behaviour is not vLLM-specific) and +an interactive terminal: the smoke test and its summary run only there. The +behaviour under test is described in +[the README](../README.md#model-serving); the automated coverage is listed in +[testing.md](testing.md#serve-smoke-test-failure). + +The repeatable route is the E2E scenario, which needs no GPU: + +```bash +cargo xtask e2e -- -n "serve-60" +``` + +By hand, a failing first chat request is hard to provoke (the engine has to pass +its readiness probe and then fail or die on the first chat request), so check the +points below when you see it, for example after an engine is killed right as it +becomes ready: + +- The heading reads `Deployment summary (smoke test failed)`, not + `Deployment summary`. +- The `status` row reads `started, smoke test failed`, not `ready`. +- `time to first token` and `throughput (approx)` read `n/a`, and a `note:` line + gives a one-line reason and names `rocm logs --service `. +- The command exits 0 and ROCm CLI does not stop the server. If the engine died + there is no server left: `rocm logs --service ` still shows its output, and + the next `rocm services list` finds the engine gone and records the service as + `stopped`, so it drops out of the default list and shows only under + `rocm services list --all`. If the engine is still up and only the chat request + failed, the service stays listed and its recorded status stays `ready`. +- `rocm serve --no-smoke-test` prints the plain `Deployment summary` with + no smoke-test note, and re-running against an already-running service prints + the `(already running)` summary with no smoke-test note. +- A reasoning model that spends its small token budget before emitting any + content is not a failure: the summary stays `ready` with the two metric rows + `n/a` and no smoke-test note. diff --git a/docs/testing.md b/docs/testing.md index db206b25d4..1f00b4a52b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4209,7 +4209,8 @@ whether a scenario may run at all (see the comment at a **prebuilt** `ROCM_CLI_BINARY`: the hook is compiled out there, the `@requires-oom-fault-injection` skip no longer applies, and the scenario fails for the wrong reason instead of being reported as skipped. Do not select it by -name on that route. Note also that `-n` matches the scenario *name* +name on that route, nor any other `@requires-oom-fault-injection` scenario +(serve-60 carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so passing an id to it silently selects nothing. @@ -4225,9 +4226,11 @@ host cannot produce. It is therefore driven by a test-only fault-injection hook: the `rocm` crate feature `e2e-oom-fault-injection`, armed at run time by `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`, makes `rocm serve` fabricate the on-disk state of a managed vLLM launch that spawned, wrote an allocator OOM traceback to the -log it owns, and never became ready. No process is spawned, and the seam -compiles out entirely without the feature, so a build that does not enable it -behaves exactly as the shipped binary does. +log it owns, and never became ready. serve-60 arms the same hook with +`ROCM_E2E_SIMULATE_OOM_LAUNCH=ready`, which fabricates a launch that did become +ready, so a mock engine can then fail its smoke test. No process is spawned, and +the seam compiles out entirely without the feature, so a build that does not +enable it behaves exactly as the shipped binary does. Only `cargo xtask e2e` enables it, by building the binary under test with `--features "rocm/e2e-test-hooks rocm/e2e-oom-fault-injection"`, and it is the @@ -4240,7 +4243,7 @@ was handed a prebuilt binary. Consequently, every lane that runs a **prebuilt** `ROCM_CLI_BINARY` — the self-hosted lanes in [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml) and the [`nightly.yml`](../.github/workflows/nightly.yml) lanes — has the hook -compiled out and resolves serve-41 to a reported **skip**, not a silent pass. It +compiled out and resolves serve-41 and serve-60 to a reported **skip**, not a silent pass. It gates PRs on the GitHub-hosted mock lane, which lets `cargo xtask e2e` build the binary itself. The prebuilt lanes' `--features` floor is asserted as a superset check rather than an exact match for this reason; see the test notes on @@ -4250,11 +4253,75 @@ check rather than an exact match for this reason; see the test notes on Be precise about *which* gate produces the skip, because the hook is not always the one that decides. On the Windows lifecycle lane in [`ci.yml`](../.github/workflows/ci.yml) the hook is compiled out too, but -serve-41 is never selected there at all: that lane runs `E2E_ONLY_LIFECYCLE` and -the scenario carries `@requires-os:linux`. On the GPU lanes it is +serve-41 and serve-60 are never selected there at all: that lane runs +`E2E_ONLY_LIFECYCLE` and the scenarios carry `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. +## Serve Smoke Test Failure + +After a fresh launch becomes ready, the `rocm serve` interactive summary sends +one small chat request. When that request cannot complete the summary is headed +`Deployment summary (smoke test failed)`, shows the status as +`started, smoke test failed` instead of `ready`, and a hedged note gives the +reason and names `rocm logs --service `. See +[the README](../README.md#model-serving) for the user-facing contract. The +launch itself stays best-effort: exit code and service record do not change, and +`--no-smoke-test` and an already-running service are unchanged. + +What is detected, and what is not: + +- Detected (the summary says the smoke test failed): a transport error or timeout, + an HTTP status other than 200, a parse error, and a chunked or `Content-Length` + stream cut short. +- Not detected (the summary reads as before, metrics `n/a`): an engine error sent + as a `data:` payload inside a `200` stream; a `200` answer with no `data:` line + at all (HTML, plain JSON, an empty body); a close-delimited stream cut after a + token; and a complete stream with no `content` token, which is deliberately not + a failure (a reasoning model can spend its whole small budget first, as the + readiness probe also allows for). +- Known limitation: the reason shown is whatever the provider call reported, and + two of its errors are not the engine's. It re-lists `/v1/models` before the + request and can fail with "no ready managed service", and it writes an audit + event after the stream is consumed. Either turns into a failed smoke test, which + is why the note says the smoke test could not complete and only that the server + may not be able to serve requests. + +Focused unit coverage: + +```bash +cargo test -p rocm --bin rocm serve_smoke_test +cargo test -p rocm --bin rocm a_failed_smoke_test_is_not_presented_as_ready_and_says_why +cargo test -p rocm --bin rocm a_passing_or_skipped_smoke_test_adds_no_failure_note +cargo test -p rocm --bin rocm an_already_running_summary_ignores_a_smoke_failure +cargo test -p rocm --bin rocm failure_reason_ +cargo test -p rocm --bin rocm the_commands_the_failure_note_names_exist_and_parse +``` + +The `serve_smoke_test` tests in [providers.rs](../apps/rocm/src/providers.rs) +run the real smoke test against a local mock engine over a socket (an engine that +dies mid-answer or inside a chunk, a 500, 401 and 404, a hostile status line, a +complete stream with no content, a close-delimited stream cut after a token). Only +the mid-stream-death test goes on to render the summary from the real outcome; the +others assert on the outcome the smoke test returns. + +- serve-60, `@id:serve-smoke-failure-summary` — a launch that becomes ready + whose engine fails its first chat request is shown as a failed smoke test, with + the engine's own reason and the full `rocm logs --service ` command, and + the engine received exactly one chat request. It carries `@requires-no-gpu`, + `@requires-os:linux` and `@requires-oom-fault-injection`: it uses the same + test-only hook as serve-41 (see [Serve OOM Guidance](#serve-oom-guidance)) in its + `ready` mode, a ready record pointing at a mock engine whose chat requests all + fail with HTTP 500, so it has serve-41's gates: against a prebuilt + `ROCM_CLI_BINARY` the hook is compiled out and the harness reports it as a skip, + unless it is selected with `-n` or `--tags`, which bypasses that skip and makes + it fail for the wrong reason. Two of its steps are hook hygiene rather than checks of the + fix: the unchanged service record and the no-daemon step. + +```bash +cargo xtask e2e -- -n "serve-60" +``` + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 99e9ede713..d871ab21d1 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -145,7 +145,7 @@ Scenarios carry stable-id and capability tags: | `@requires-case-sensitive-fs` | Premise is two files whose names differ only in letter case. Resolves to **skip** where the scenarios' temp root folds case. The capability probe's `case_sensitive_fs` writes two such names in its own temp dir, which follows `$TMPDIR` like every scenario's isolated root, and checks they stayed two files. `@requires-os:linux` cannot express this: `$TMPDIR` on Linux, WSL2 included, can sit on a casefolded directory or a mounted Windows drive. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | -| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch that runs out of GPU memory so the OOM guidance can be verified without a GPU. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | +| `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch so `rocm serve`'s summary can be verified without a GPU: one that runs out of GPU memory (the OOM guidance), or, in its `ready` mode, one that becomes ready and then fails its smoke test. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | | `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios, the native-Windows vLLM refusal scenarios serve-33/34/35, and the side-by-side runtime scenarios runtime-lifecycle-19/20/21. | diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a67d286c65..29ee3d620c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -17,7 +17,7 @@ Feature: Model serving # This discriminates nothing yet: the scenario is an expected failure on every # host while its xfail row stands, and the vLLM run stops before printing a # plan, so the step panics there whatever the lemonade half did. It becomes - # regression coverage when that row is narrowed. serve-55 and the unit tests in + # regression coverage when that row is narrowed. serve-60 and the unit tests in # serve_cmd.rs are what hold the per-engine refs in the meantime. @id:serve-short-name-consistent-across-engines Scenario: serve-02 - Every engine expands a short name rather than serving it @@ -963,3 +963,19 @@ Feature: Model serving Given a managed runtime is active When the user serves a model that requires an API key Then the server's API key is kept where only that user can read it + + # A launch that became ready but whose first chat request fails must not be + # shown as a healthy deployment (#523). Same test-only hook as serve-41, in its + # `ready` mode, pointed at a mock engine that fails every chat request. The + # unit and provider tests prove the rendering and the failure classes; this + # proves `serve()` carries the smoke test's failure into the summary. + @id:serve-smoke-failure-summary @requires-no-gpu @requires-os:linux @requires-oom-fault-injection + Scenario: serve-60 - A launch whose first chat request fails is not presented as ready + Given a managed vLLM launch that becomes ready but whose engine fails its first chat request + When the user opens the interactive serve summary against that engine + Then the deployment summary says the smoke test failed and does not call the service ready + And the deployment summary gives the engine's own reason + And the engine received exactly one chat request + And the deployment summary names the logs command for that service + And the launch's service record is left as ready + And no background automation daemon was left behind diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index d63549166f..b586864c72 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -149,7 +149,7 @@ pub struct HostCapability { pub platform_slug: String, /// Whether the `rocm` binary under test carries the test-only /// `e2e-oom-fault-injection` hook, so `@requires-oom-fault-injection` - /// scenarios can simulate a GPU-less OOM launch. It is compiled out of the + /// scenarios (serve-41, serve-60) can simulate a GPU-less launch. It is compiled out of the /// shipping binary, and the harness cannot probe for it (no product command /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with @@ -1442,8 +1442,8 @@ Local model engines /// `expectation.rs`'s `requires_oom_fault_injection_runs_only_when_the_hook_is_built_in` /// builds `HostCapability { oom_fault_injection: .. }` by hand and so never /// reaches this comparison; without this test, editing the literal on one - /// side of the producer/consumer pair would make serve-41 a permanent skip - /// with nothing red. The `"1"` asserted here is the same literal + /// side of the producer/consumer pair would make every + /// `@requires-oom-fault-injection` scenario a permanent skip with nothing red. The `"1"` asserted here is the same literal /// `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned /// by `release_build_resolves_and_exports_both_cli_binaries` on that side. #[test] diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6b..a9ace74dc6 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -116,7 +116,8 @@ pub struct ScenarioDecl { pub requires_wsl: bool, /// `@requires-oom-fault-injection`: the scenario drives the test-only /// `e2e-oom-fault-injection` hook (armed by `ROCM_E2E_SIMULATE_OOM_LAUNCH`) - /// to fabricate a GPU-less OOM launch, so it can only run against a binary + /// to fabricate a GPU-less launch (serve-41: it OOMs; serve-60: it becomes + /// ready and its smoke test fails), so it can only run against a binary /// compiled with that feature. `xtask e2e` builds it in when it builds the /// binary itself (the GitHub-hosted mock lane), but the self-hosted lanes /// test a prebuilt shipping release binary that has the hook compiled out — diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index e419365c3e..87c0092189 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -52,9 +52,9 @@ const UNRELATED_LIVE_MODEL: &str = "e2e/unrelated-live-model"; /// [`REUSE_REFUSAL_MODEL`] picked it, so this aliases that constant rather than /// repeating the literal. const LEMONADE_SERVE_TARGET: &str = REUSE_REFUSAL_MODEL; -/// Model id for the OOM-launch scenario, kept distinct from every other e2e -/// serve model so its service records can never collide with another -/// scenario's. +/// Model id for the fabricated-launch scenarios (serve-41 and serve-60), kept +/// distinct from every other e2e serve model so their service records can never +/// collide with another scenario's. const OOM_LAUNCH_MODEL: &str = "e2e/oom-launch-model"; /// How long to wait for a freshly served model's endpoint to become ready. /// @@ -3220,6 +3220,164 @@ async fn assert_oom_launch_quotes_the_real_line(world: &mut E2eWorld) { ); } +/// A managed vLLM launch that became ready, whose engine (a mock) answers +/// `/v1/models` but fails every chat request with HTTP 500. The launch is +/// fabricated by the `ready` mode of the same test-only hook serve-41 arms; the +/// mock's port is passed to `rocm serve` by the `When` below, which is how the +/// fabricated record points the post-launch smoke test at it. +#[given("a managed vLLM launch that becomes ready but whose engine fails its first chat request")] +async fn plant_ready_launch_with_failing_engine(world: &mut E2eWorld) { + // An empty chat script means this engine always fails chat: `/v1/models` + // answers, and every chat request is "past the end of the script" and gets + // HTTP 500. + let mock = MockServer::start_with_chat_script(OOM_LAUNCH_MODEL, Vec::new()).await; + world.mock = Some(mock); + // Consumed by `e2e_simulate_oom_launch` in + // `apps/rocm/src/service_launch/readiness.rs`; the name is spelled out on both + // sides because the binary's crate cannot depend on this harness. + world + .command_env + .push(("ROCM_E2E_SIMULATE_OOM_LAUNCH", "ready".into())); +} + +#[when("the user opens the interactive serve summary against that engine")] +async fn open_summary_against_failing_engine(world: &mut E2eWorld) { + let port = world + .mock + .as_ref() + .expect("no mock engine was started") + .port() + .to_string(); + let mut session = TuiSession::spawn( + world, + &[ + "serve", + OOM_LAUNCH_MODEL, + "--engine", + "vllm", + "--env-id", + "e2e-smoke-failure", + "--port", + &port, + ], + ) + .unwrap_or_else(|error| panic!("failed to open interactive serve summary: {error}")); + // `wait_for_exit` requires exit code 0: a failed smoke test must not fail + // the launch. + session + .wait_for_exit(interactive_summary_timeout()) + .await + .unwrap_or_else(|error| panic!("interactive serve summary failed: {error}")); + world.tui = Some(session); +} + +#[then("the deployment summary says the smoke test failed and does not call the service ready")] +async fn assert_summary_reports_failed_smoke_test(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains("Deploymentsummary(smoketestfailed)"), + "the heading must say the smoke test failed:\n{screen}" + ); + assert!( + flattened.contains("started,smoketestfailed"), + "the status row must not read `ready`:\n{screen}" + ); + assert!( + !flattened.contains("statusready"), + "a failed smoke test must not leave `status ready` on screen:\n{screen}" + ); +} + +#[then("the deployment summary gives the engine's own reason")] +async fn assert_summary_gives_engine_reason(world: &mut E2eWorld) { + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + // The exact reason the provider reports for the mock's answer, not just a + // `500` that the ephemeral port or a timestamp-based service id could also + // contain, and not the reason of some other failed outcome (for example "no + // ready managed service"). + assert!( + flattened.contains("thesmoketestcouldnotcomplete") + && flattened.contains("localproviderreturnedHTTP/1.1500InternalServerError"), + "the note must carry the engine's HTTP 500 as the reason:\n{screen}" + ); +} + +#[then("the engine received exactly one chat request")] +async fn assert_engine_received_one_chat_request(world: &mut E2eWorld) { + let received = world + .mock + .as_ref() + .expect("no mock engine was started") + .chat_requests() + .len(); + assert_eq!( + received, 1, + "the smoke test must send exactly one chat request to the engine" + ); +} + +#[then("the deployment summary names the logs command for that service")] +async fn assert_summary_names_logs_command(world: &mut E2eWorld) { + let service_id = launched_service_record(world)["service_id"] + .as_str() + .expect("the service record has no service_id") + .to_owned(); + let screen = world + .tui + .as_ref() + .expect("no interactive serve summary") + .screen_text(); + let flattened = screen_without_whitespace(&screen); + assert!( + flattened.contains(&format!("check`rocmlogs--service{service_id}`")), + "the note must name `rocm logs --service {service_id}` in full:\n{screen}" + ); +} + +/// Hook hygiene, not a check of the fix: it holds the "launch stays best-effort, +/// the service record is not changed" constraint, and passes with the fix +/// reverted. The failure reaching the summary is what the steps above assert. +#[then("the launch's service record is left as ready")] +async fn assert_service_record_unchanged(world: &mut E2eWorld) { + assert_eq!( + launched_service_record(world)["status"], + "ready", + "a failed smoke test must not change the service record" + ); +} + +/// The one service record the fabricated launch wrote. +fn launched_service_record(world: &E2eWorld) -> serde_json::Value { + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let mut records: Vec = std::fs::read_dir(&services) + .expect("the launch left no services directory") + .filter_map(Result::ok) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .map(|entry| { + serde_json::from_slice(&std::fs::read(entry.path()).expect("record is readable")) + .expect("record is JSON") + }) + .collect(); + assert_eq!(records.len(), 1, "expected exactly the launched service"); + records.remove(0) +} + /// Collapse a rendered PTY screen to its non-whitespace characters, so an /// assertion survives the terminal soft-wrapping a long line across grid rows /// (which inserts row breaks mid-phrase / mid-token). Only meaningful for @@ -3824,9 +3982,12 @@ async fn serve_wait_refused(world: &mut E2eWorld) { } /// The discriminating assertion for the fabricated-launch daemon suppression. +/// Hook hygiene, shared by serve-41 and serve-60: it checks the test seam, not +/// the behaviour either scenario is about. /// -/// `simulate_oom_managed_launch` writes a `starting` record for a process that -/// never existed, so `ensure_background_helper_running_quiet` would leave a +/// `simulate_oom_managed_launch` writes a record (`starting`, or `ready` in its +/// `ready` mode) for a process that never existed, so +/// `ensure_background_helper_running_quiet` would leave a /// `rocm daemon` supervising a pid that will never appear — and nothing in this /// scenario's teardown stops it. Observed in the wild as daemons outliving their /// e2e checkout by days. @@ -3854,7 +4015,7 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { assert_ne!( state.get("running").and_then(serde_json::Value::as_bool), Some(true), - "a fabricated OOM launch must not start the automation daemon, but \ + "a fabricated launch must not start the automation daemon, but \ {} says one is running:\n{text}", state_path.display() ); @@ -3863,7 +4024,7 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let claim_path = root.join("autostart.claim"); assert!( !claim_path.is_file(), - "a fabricated OOM launch must not even claim a daemon autostart, but {} exists: {:?}", + "a fabricated launch must not even claim a daemon autostart, but {} exists: {:?}", claim_path.display(), std::fs::read_to_string(&claim_path).ok() ); diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 1d8cd0e531..42266bcbc9 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -106,8 +106,9 @@ pub fn run(args: &[String]) -> Result<()> { if binaries.build_release { let status = Command::new(&cargo) // `--features rocm/e2e-oom-fault-injection` compiles in the test-only - // `rocm serve` OOM-launch fault injection used by the positive - // `serve-oom-launch-memory-guidance` scenario. It is scoped to the + // `rocm serve` launch fault injection (an OOM launch, or a ready + // launch) used by every `@requires-oom-fault-injection` scenario + // (serve-41 and serve-60). It is scoped to the // `rocm` package (rocmd has no such feature) and to this e2e build // only — the release build never enables it, so shipped binaries // carry no fault-injection hook. It joins `rocm/e2e-test-hooks` From 0dd7af818fa67f16a720967b7daae6df9075aa0e Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 11:41:33 +0000 Subject: [PATCH 3/4] docs(serve): say what the smoke-failure note does and does not detect (ROCMAI-703) The testing guide claimed a close-delimited stream cut after a token shows `n/a` metrics; it shows the metrics measured up to the cut. The README and the manual test steps now point at the guide's list of failure classes that are not detected, so they no longer imply the note catches everything. The failure reason no longer lets an engine add a backtick-quoted command next to the note's own logs hint. A test pins the note's "server may not be able to serve requests" warning. Comments no longer list scenario ids that drift. The guide records that two ready services on one model can make the verdict describe the other service. The earlier commit's message refers to the e2e scenario as serve-55; it is serve-56 after merging main. Signed-off-by: Tomas Saaristola --- README.md | 5 +++- apps/rocm/src/serve_cmd.rs | 4 +-- apps/rocm/src/serve_summary.rs | 38 +++++++++++++++++++++++---- docs/manual-testing.md | 7 ++++- docs/testing.md | 18 ++++++++----- tests/e2e-cucumber/src/capability.rs | 12 ++++----- tests/e2e-cucumber/src/expectation.rs | 4 +-- xtask/src/e2e.rs | 6 ++--- 8 files changed, 68 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index f8927a3f3f..e0e91b892e 100644 --- a/README.md +++ b/README.md @@ -1267,7 +1267,10 @@ interactive terminal, if the smoke test cannot complete, the summary is headed `(smoke test failed)`, shows the status as `started, smoke test failed` instead of `ready`, and a note gives the reason and points at `rocm logs --service `. ROCm CLI does not stop the server, and the command -still exits 0. Control +still exits 0. Which failures are detected, and which still read as `ready`, is +listed under +[Serve Smoke Test Failure](https://github.com/ROCm/rocm-cli/blob/main/docs/testing.md#serve-smoke-test-failure) +in the testing guide. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index d56eae3824..523d105599 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -864,8 +864,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // (probe returns `None`). The E2E-only // backend-failure scenario bypasses this host precondition so the black-box // test reaches Lemonade's backend boundary without real GPU hardware, and the - // E2E-only OOM-launch fault injection likewise stands in for the GPU it does - // not have. + // E2E-only launch fault injection (either mode) likewise stands in for the + // GPU it does not have. // // The scripted free-space override opens the same waiver: it exists to refuse a // Lemonade install on a drive that cannot hold it, which `rocm serve` only diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 9270564add..6fbc077f78 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -114,10 +114,13 @@ const SMOKE_FAILURE_MAX_CHARS: usize = 200; /// Reduces an error message to one printable line. On the smoke-test path the /// engine-controlled text in it is the HTTP status line, reason phrase included -/// (the local provider reports `local provider returned {status_line}`, never the -/// response body), so it is untrusted. Terminal escape sequences and any control -/// or Unicode format character (a bidi override, a zero-width mark) are dropped, -/// line breaks and runs of whitespace become a single space, and the result is +/// (the local provider reports `local provider returned {status_line}`, not the +/// response body), and the chunk-size line it quotes when a chunked body is +/// malformed (`invalid local provider chunk size`), so it is untrusted. Terminal +/// escape sequences and any control or Unicode format character (a bidi override, +/// a zero-width mark) are dropped, line breaks and runs of whitespace become a +/// single space, backticks become apostrophes (the note quotes its own command in +/// backticks, and the reason must not add a second one), and the result is /// length-capped. fn one_line_reason(raw: &str) -> String { // The escape scanner, not this function, decides where a line break falls: @@ -127,7 +130,8 @@ fn one_line_reason(raw: &str) -> String { .join(" ") .split_whitespace() .collect::>() - .join(" "); + .join(" ") + .replace('`', "'"); if collapsed.chars().count() > SMOKE_FAILURE_MAX_CHARS { let mut cut: String = collapsed.chars().take(SMOKE_FAILURE_MAX_CHARS).collect(); cut.push_str("..."); @@ -1281,6 +1285,9 @@ mod tests { // That the named commands exist and parse is // `the_commands_the_failure_note_names_exist_and_parse`. assert!(rendered.contains("check `rocm logs --service vllm-qwen-1720000000`")); + // The hedge: the note must not claim the server is broken, only that it + // may be unable to serve. + assert!(rendered.contains("; the server may not be able to serve requests")); } #[test] @@ -1336,6 +1343,27 @@ mod tests { assert_eq!(one_line_reason("short"), "short"); } + #[test] + fn failure_reason_cannot_add_a_backtick_quoted_command_to_the_note() { + // A chunk-size line the engine sent is quoted in the provider's error. + let reason = one_line_reason("invalid local provider chunk size `run `rocm stop` now`"); + assert!(!reason.contains('`')); + assert_eq!( + reason, + "invalid local provider chunk size 'run 'rocm stop' now'" + ); + let rendered = render_summary(&failed_summary(&reason)); + let note = rendered + .lines() + .find(|line| line.contains("the smoke test could not complete")) + .expect("failure note present"); + // Constant message: `note` derives from a type that carries an API key. + assert!( + note.split('`').skip(1).step_by(2).count() == 1, + "the note must hold exactly one backtick-quoted command" + ); + } + #[test] fn failure_reason_keeps_the_text_after_a_cut_off_escape_sequence() { // A capture cut mid-sequence must not eat the first real character after diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 984ef8651d..8a8906c73d 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1950,7 +1950,8 @@ behaviour under test is described in [the README](../README.md#model-serving); the automated coverage is listed in [testing.md](testing.md#serve-smoke-test-failure). -The repeatable route is the E2E scenario, which needs no GPU: +The repeatable route is the E2E scenario, which needs no GPU but runs on Linux +and WSL only (`@requires-os:linux`): ```bash cargo xtask e2e -- -n "serve-60" @@ -1975,6 +1976,10 @@ becomes ready: - `rocm serve --no-smoke-test` prints the plain `Deployment summary` with no smoke-test note, and re-running against an already-running service prints the `(already running)` summary with no smoke-test note. +- Some failures are deliberately not detected and still read as `ready` (an error + sent as a `data:` payload inside a `200` stream, a `200` answer with no `data:` + line, a close-delimited stream cut after a token); the full list is in + [testing.md](testing.md#serve-smoke-test-failure). - A reasoning model that spends its small token budget before emitting any content is not a failure: the summary stays `ready` with the two metric rows `n/a` and no smoke-test note. diff --git a/docs/testing.md b/docs/testing.md index 1f00b4a52b..ce293203fe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4274,18 +4274,24 @@ What is detected, and what is not: - Detected (the summary says the smoke test failed): a transport error or timeout, an HTTP status other than 200, a parse error, and a chunked or `Content-Length` stream cut short. -- Not detected (the summary reads as before, metrics `n/a`): an engine error sent - as a `data:` payload inside a `200` stream; a `200` answer with no `data:` line - at all (HTML, plain JSON, an empty body); a close-delimited stream cut after a - token; and a complete stream with no `content` token, which is deliberately not - a failure (a reasoning model can spend its whole small budget first, as the - readiness probe also allows for). +- Not detected (no failure is reported and the status reads `ready`): an engine + error sent as a `data:` payload inside a `200` stream; a `200` answer with no + `data:` line at all (HTML, plain JSON, an empty body); a close-delimited stream + cut after a token; and a complete stream with no `content` token, which is + deliberately not a failure (a reasoning model can spend its whole small budget + first, as the readiness probe also allows for). The metric rows read `n/a` for + all of these except the close-delimited cut, which shows the metrics measured + up to the cut. - Known limitation: the reason shown is whatever the provider call reported, and two of its errors are not the engine's. It re-lists `/v1/models` before the request and can fail with "no ready managed service", and it writes an audit event after the stream is consumed. Either turns into a failed smoke test, which is why the note says the smoke test could not complete and only that the server may not be able to serve requests. +- Known limitation: the request goes to the first ready local service serving the + model, not to the service id the summary reports. With two ready services + serving the same model, the verdict can describe the other one while the note + names `rocm logs --service ` of the one just launched. Focused unit coverage: diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index b586864c72..d8857f43c0 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -148,9 +148,9 @@ pub struct HostCapability { /// "mock" (no AMD GPU), else the family/target (e.g. "mi300x", "strix-halo"). pub platform_slug: String, /// Whether the `rocm` binary under test carries the test-only - /// `e2e-oom-fault-injection` hook, so `@requires-oom-fault-injection` - /// scenarios (serve-41, serve-60) can simulate a GPU-less launch. It is compiled out of the - /// shipping binary, and the harness cannot probe for it (no product command + /// `e2e-oom-fault-injection` hook, so the scenarios tagged + /// `@requires-oom-fault-injection` can simulate a GPU-less launch. It is + /// compiled out of the shipping binary, and the harness cannot probe for it (no product command /// exposes it), so `xtask e2e` reports it via the [`OOM_FAULT_INJECTION_ENV`] /// environment variable — set only when xtask built the binary itself with /// the feature, and absent for a prebuilt `ROCM_CLI_BINARY` (the self-hosted @@ -1442,9 +1442,9 @@ Local model engines /// `expectation.rs`'s `requires_oom_fault_injection_runs_only_when_the_hook_is_built_in` /// builds `HostCapability { oom_fault_injection: .. }` by hand and so never /// reaches this comparison; without this test, editing the literal on one - /// side of the producer/consumer pair would make every - /// `@requires-oom-fault-injection` scenario a permanent skip with nothing red. The `"1"` asserted here is the same literal - /// `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned + /// side of the producer/consumer pair would make the scenarios tagged + /// `@requires-oom-fault-injection` a permanent skip with nothing red. The + /// `"1"` asserted here is the same literal `xtask::e2e`'s `command.env(OOM_FAULT_INJECTION_ENV, "1")` writes, pinned /// by `release_build_resolves_and_exports_both_cli_binaries` on that side. #[test] fn only_the_exact_value_the_producer_writes_arms_the_hook() { diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index a9ace74dc6..6771d09dcd 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -116,8 +116,8 @@ pub struct ScenarioDecl { pub requires_wsl: bool, /// `@requires-oom-fault-injection`: the scenario drives the test-only /// `e2e-oom-fault-injection` hook (armed by `ROCM_E2E_SIMULATE_OOM_LAUNCH`) - /// to fabricate a GPU-less launch (serve-41: it OOMs; serve-60: it becomes - /// ready and its smoke test fails), so it can only run against a binary + /// to fabricate a GPU-less launch (one that OOMs, or one that becomes ready + /// and then fails its smoke test), so it can only run against a binary /// compiled with that feature. `xtask e2e` builds it in when it builds the /// binary itself (the GitHub-hosted mock lane), but the self-hosted lanes /// test a prebuilt shipping release binary that has the hook compiled out — diff --git a/xtask/src/e2e.rs b/xtask/src/e2e.rs index 42266bcbc9..3b376ed73f 100644 --- a/xtask/src/e2e.rs +++ b/xtask/src/e2e.rs @@ -107,9 +107,9 @@ pub fn run(args: &[String]) -> Result<()> { let status = Command::new(&cargo) // `--features rocm/e2e-oom-fault-injection` compiles in the test-only // `rocm serve` launch fault injection (an OOM launch, or a ready - // launch) used by every `@requires-oom-fault-injection` scenario - // (serve-41 and serve-60). It is scoped to the - // `rocm` package (rocmd has no such feature) and to this e2e build + // launch) used by the scenarios tagged `@requires-oom-fault-injection`. + // It is scoped to the `rocm` package (rocmd has no such feature) and + // to this e2e build // only — the release build never enables it, so shipped binaries // carry no fault-injection hook. It joins `rocm/e2e-test-hooks` // (the general e2e hook surface) in a single space-separated From bdd4252a7a3decb39322880e9ebf8e3acdc8cf8b Mon Sep 17 00:00:00 2001 From: Tomas Saaristola Date: Fri, 13 Nov 2026 16:19:29 +0000 Subject: [PATCH 4/4] docs(serve): correct what the serve-57 steps and the smoke-failure notes claim (ROCMAI-703) A review found claims the code cannot back. The daemon-left-behind step cannot discriminate the autostart guard because the isolated config enables no watcher. The testing guide overstated `n/a` metrics for undetected classes and described a two-ready-services case that cannot happen, while omitting the real false pass: the smoke request is routed by model name, not to the launched service. A length-cap boundary had no test. The text now says what the code does, the cap is tested at 200/201 characters, a load-bearing line in the ready hook is commented, and prose cites @id tags instead of scenario numbers that had already drifted. Signed-off-by: Tomas Saaristola --- apps/rocm/Cargo.toml | 5 +- apps/rocm/src/serve_cmd.rs | 9 ++- apps/rocm/src/serve_summary.rs | 15 +++++ apps/rocm/src/service_launch/readiness.rs | 2 + docs/manual-testing.md | 4 +- docs/testing.md | 62 ++++++++++++------- .../features/model_serving.feature | 3 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 40 +++++++----- 8 files changed, 97 insertions(+), 43 deletions(-) diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 4b72d65640..83567da2b2 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -21,8 +21,9 @@ e2e-test-hooks = [ # mock-lane build (never by the release build), it compiles in a `rocm serve` # hook that simulates a managed vLLM launch, so the `rocm serve` summary can be # verified on a GPU-less CI host: one that OOM'd and never became ready (the -# OOM guidance), or, in the hook's `ready` mode, one that became ready and then -# fails its smoke test. +# OOM guidance), or, in the hook's `ready` mode, one that became ready (the +# hook only fabricates the ready record; a mock engine supplies the failure of +# the post-launch smoke test). # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in # `service_launch/readiness.rs`. e2e-oom-fault-injection = [] diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 523d105599..f359699db3 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1204,9 +1204,12 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // runs *before* the OOM note is computed in the `summary_mode` block // below, so the previous claim here, that the note "is computed before // it would have run", was backwards and could not have excused it. - // The `no background automation daemon was left behind` step of serve-41 - // and serve-55 is what holds the branch: deleting it left the scenario - // green with a `rocm daemon` still running. + // The `no background automation daemon was left behind` step of the + // `@id:serve-oom-launch-memory-guidance` and + // `@id:serve-smoke-failure-summary` scenarios records that none was + // started but does not discriminate this branch: the helper starts only + // when a watcher is enabled, and the isolated config those scenarios use + // enables none, so removing the branch leaves them green. // // `e2e_simulate_oom_launch` is a `const fn` returning `false` without // the test-only feature, so this branch folds away in shipped builds. diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 6fbc077f78..c30bcc654a 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -1343,6 +1343,21 @@ mod tests { assert_eq!(one_line_reason("short"), "short"); } + #[test] + fn failure_reason_cap_holds_at_its_boundary() { + // Exactly the cap is kept whole; one more is cut and marked. + let at_cap = "y".repeat(SMOKE_FAILURE_MAX_CHARS); + assert!( + one_line_reason(&at_cap) == at_cap, + "a reason of exactly the cap must be kept unchanged" + ); + let over = "y".repeat(SMOKE_FAILURE_MAX_CHARS + 1); + assert!( + one_line_reason(&over) == format!("{at_cap}..."), + "a reason one character over the cap must be cut to the cap plus an ellipsis" + ); + } + #[test] fn failure_reason_cannot_add_a_backtick_quoted_command_to_the_note() { // A chunk-size line the engine sent is quoted in the provider's error. diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 799617553f..c10a29eea6 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -256,6 +256,8 @@ pub(super) fn simulate_oom_managed_launch( // provider needs to send the smoke test to `host:port`; no log is written // because nothing here failed to start. record.status = "ready".to_owned(); + // Load-bearing: without it the smoke test's `/v1/models` listing probe + // fails and the reason becomes "no ready managed service". record.inference_verified_at_unix_ms = Some(1); record.write()?; return Ok(ManagedLaunchReport { diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 8a8906c73d..ec2b270669 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1957,6 +1957,8 @@ and WSL only (`@requires-os:linux`): cargo xtask e2e -- -n "serve-60" ``` +(`@id:serve-smoke-failure-summary`.) + By hand, a failing first chat request is hard to provoke (the engine has to pass its readiness probe and then fail or die on the first chat request), so check the points below when you see it, for example after an engine is killed right as it @@ -1970,7 +1972,7 @@ becomes ready: - The command exits 0 and ROCm CLI does not stop the server. If the engine died there is no server left: `rocm logs --service ` still shows its output, and the next `rocm services list` finds the engine gone and records the service as - `stopped`, so it drops out of the default list and shows only under + `stopped` (or `failed` for AIM), so it drops out of the default list and shows only under `rocm services list --all`. If the engine is still up and only the chat request failed, the service stays listed and its recorded status stays `ready`. - `rocm serve --no-smoke-test` prints the plain `Deployment summary` with diff --git a/docs/testing.md b/docs/testing.md index ce293203fe..2271a188ff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4210,7 +4210,7 @@ a **prebuilt** `ROCM_CLI_BINARY`: the hook is compiled out there, the `@requires-oom-fault-injection` skip no longer applies, and the scenario fails for the wrong reason instead of being reported as skipped. Do not select it by name on that route, nor any other `@requires-oom-fault-injection` scenario -(serve-60 carries the same tag). Note also that `-n` matches the scenario *name* +(`@id:serve-smoke-failure-summary` carries the same tag). Note also that `-n` matches the scenario *name* (`serve-41 - A launch that runs out of GPU memory ...`), not the `@id:` tag, so passing an id to it silently selects nothing. @@ -4221,12 +4221,12 @@ passing an id to it silently selects nothing. ### The OOM fault-injection capability -serve-41 needs a managed launch that really failed on VRAM, which a GPU-less CI +The `@id:serve-oom-launch-memory-guidance` scenario needs a managed launch that really failed on VRAM, which a GPU-less CI host cannot produce. It is therefore driven by a test-only fault-injection hook: the `rocm` crate feature `e2e-oom-fault-injection`, armed at run time by `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`, makes `rocm serve` fabricate the on-disk state of a managed vLLM launch that spawned, wrote an allocator OOM traceback to the -log it owns, and never became ready. serve-60 arms the same hook with +log it owns, and never became ready. `@id:serve-smoke-failure-summary` arms the same hook with `ROCM_E2E_SIMULATE_OOM_LAUNCH=ready`, which fabricates a launch that did become ready, so a mock engine can then fail its smoke test. No process is spawned, and the seam compiles out entirely without the feature, so a build that does not @@ -4243,7 +4243,7 @@ was handed a prebuilt binary. Consequently, every lane that runs a **prebuilt** `ROCM_CLI_BINARY` — the self-hosted lanes in [`e2e-selfhosted.yml`](../.github/workflows/e2e-selfhosted.yml) and the [`nightly.yml`](../.github/workflows/nightly.yml) lanes — has the hook -compiled out and resolves serve-41 and serve-60 to a reported **skip**, not a silent pass. It +compiled out and resolves both of those scenarios to a reported **skip**, not a silent pass. It gates PRs on the GitHub-hosted mock lane, which lets `cargo xtask e2e` build the binary itself. The prebuilt lanes' `--features` floor is asserted as a superset check rather than an exact match for this reason; see the test notes on @@ -4253,7 +4253,7 @@ check rather than an exact match for this reason; see the test notes on Be precise about *which* gate produces the skip, because the hook is not always the one that decides. On the Windows lifecycle lane in [`ci.yml`](../.github/workflows/ci.yml) the hook is compiled out too, but -serve-41 and serve-60 are never selected there at all: that lane runs +both scenarios are never selected there at all: that lane runs `E2E_ONLY_LIFECYCLE` and the scenarios carry `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. @@ -4279,19 +4279,24 @@ What is detected, and what is not: `data:` line at all (HTML, plain JSON, an empty body); a close-delimited stream cut after a token; and a complete stream with no `content` token, which is deliberately not a failure (a reasoning model can spend its whole small budget - first, as the readiness probe also allows for). The metric rows read `n/a` for - all of these except the close-delimited cut, which shows the metrics measured - up to the cut. + first, as the readiness probe also allows for). The metric rows read `n/a` when + no content token arrived first; tokens that did arrive (the close-delimited cut, + or an in-band `data:` error after content) are measured. - Known limitation: the reason shown is whatever the provider call reported, and - two of its errors are not the engine's. It re-lists `/v1/models` before the - request and can fail with "no ready managed service", and it writes an audit - event after the stream is consumed. Either turns into a failed smoke test, which - is why the note says the smoke test could not complete and only that the server - may not be able to serve requests. -- Known limitation: the request goes to the first ready local service serving the - model, not to the service id the summary reports. With two ready services - serving the same model, the verdict can describe the other one while the note - names `rocm logs --service ` of the one just launched. + some of its errors are not the engine's, for example the `/v1/models` re-list + before the request ("no ready managed service"), an unreadable services-directory + entry, a rejected endpoint URL, and the audit event written after the stream is + consumed. Each turns into a failed smoke test, which is why the note says the + smoke test could not complete and only that the server may not be able to serve + requests. +- Known limitation: the smoke request is routed by model name, not to the launched + service. The newest ready service serving the model is used, so normally the one + just launched. If the new service fails the 2 s `/v1/models` listing probe (for + example the engine was killed just as it became ready) it is filtered out: an + older ready service with the same model then answers, and the summary shows + `ready` with that service's metrics and no note, a false pass on a dead launch. + With no older service the reason shown is rocm's "no ready managed service", not + the engine's. Focused unit coverage: @@ -4311,18 +4316,31 @@ complete stream with no content, a close-delimited stream cut after a token). On the mid-stream-death test goes on to render the summary from the real outcome; the others assert on the outcome the smoke test returns. -- serve-60, `@id:serve-smoke-failure-summary` — a launch that becomes ready +- `@id:serve-smoke-failure-summary` (serve-60) — a launch that becomes ready whose engine fails its first chat request is shown as a failed smoke test, with the engine's own reason and the full `rocm logs --service ` command, and the engine received exactly one chat request. It carries `@requires-no-gpu`, `@requires-os:linux` and `@requires-oom-fault-injection`: it uses the same - test-only hook as serve-41 (see [Serve OOM Guidance](#serve-oom-guidance)) in its + test-only hook as `@id:serve-oom-launch-memory-guidance` (see [Serve OOM Guidance](#serve-oom-guidance)) in its `ready` mode, a ready record pointing at a mock engine whose chat requests all - fail with HTTP 500, so it has serve-41's gates: against a prebuilt + fail with HTTP 500, so it has that scenario's gates: against a prebuilt `ROCM_CLI_BINARY` the hook is compiled out and the harness reports it as a skip, unless it is selected with `-n` or `--tags`, which bypasses that skip and makes - it fail for the wrong reason. Two of its steps are hook hygiene rather than checks of the - fix: the unchanged service record and the no-daemon step. + it fail for the wrong reason. Some of its steps are not checks of the fix, so say + plainly what they prove: + - The exit code 0 is asserted in the `When` step (`wait_for_exit`), not in a + `Then` step. + - "the engine received exactly one chat request" passes with the fix reverted + (the smoke request was already sent); it shows the request was not retried. + - "the engine's own reason" matches rocm's wording of the status line around the + engine's HTTP status, so it pins that phrasing as well. + - "the launch's service record is left as ready" passes with the fix reverted; + it is the only assertion behind the "ROCm CLI does not stop the server" + promise in the README and `rocm serve --help`. + - "no background automation daemon was left behind" is hygiene only: it records + that no daemon was started but does not discriminate the + `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs`, because no watcher is + enabled in the isolated config and the helper starts only when one is. ```bash cargo xtask e2e -- -n "serve-60" diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 29ee3d620c..096bd0825c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -965,7 +965,8 @@ Feature: Model serving Then the server's API key is kept where only that user can read it # A launch that became ready but whose first chat request fails must not be - # shown as a healthy deployment (#523). Same test-only hook as serve-41, in its + # shown as a healthy deployment (#523). Same test-only hook as + # @id:serve-oom-launch-memory-guidance, in its # `ready` mode, pointed at a mock engine that fails every chat request. The # unit and provider tests prove the rendering and the failure classes; this # proves `serve()` carries the smoke test's failure into the summary. diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 87c0092189..990bb6b188 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -52,7 +52,8 @@ const UNRELATED_LIVE_MODEL: &str = "e2e/unrelated-live-model"; /// [`REUSE_REFUSAL_MODEL`] picked it, so this aliases that constant rather than /// repeating the literal. const LEMONADE_SERVE_TARGET: &str = REUSE_REFUSAL_MODEL; -/// Model id for the fabricated-launch scenarios (serve-41 and serve-60), kept +/// Model id for the fabricated-launch scenarios (`@id:serve-oom-launch-memory-guidance` +/// and `@id:serve-smoke-failure-summary`), kept /// distinct from every other e2e serve model so their service records can never /// collide with another scenario's. const OOM_LAUNCH_MODEL: &str = "e2e/oom-launch-model"; @@ -3222,7 +3223,8 @@ async fn assert_oom_launch_quotes_the_real_line(world: &mut E2eWorld) { /// A managed vLLM launch that became ready, whose engine (a mock) answers /// `/v1/models` but fails every chat request with HTTP 500. The launch is -/// fabricated by the `ready` mode of the same test-only hook serve-41 arms; the +/// fabricated by the `ready` mode of the same test-only hook that +/// `@id:serve-oom-launch-memory-guidance` arms; the /// mock's port is passed to `rocm serve` by the `When` below, which is how the /// fabricated record points the post-launch smoke test at it. #[given("a managed vLLM launch that becomes ready but whose engine fails its first chat request")] @@ -3293,6 +3295,9 @@ async fn assert_summary_reports_failed_smoke_test(world: &mut E2eWorld) { ); } +/// Matches rocm's own wording of the status line (`the smoke test could not +/// complete`, `local provider returned ...`) around the engine's HTTP status, so +/// it pins that phrasing as well as the engine's reason. #[then("the deployment summary gives the engine's own reason")] async fn assert_summary_gives_engine_reason(world: &mut E2eWorld) { let screen = world @@ -3312,6 +3317,9 @@ async fn assert_summary_gives_engine_reason(world: &mut E2eWorld) { ); } +/// Passes with the fix reverted: the smoke request was sent before the fix too. +/// It proves only that the summary's request reached the mock once and was not +/// retried, so the failure the summary shows is that one request's. #[then("the engine received exactly one chat request")] async fn assert_engine_received_one_chat_request(world: &mut E2eWorld) { let received = world @@ -3344,9 +3352,11 @@ async fn assert_summary_names_logs_command(world: &mut E2eWorld) { ); } -/// Hook hygiene, not a check of the fix: it holds the "launch stays best-effort, -/// the service record is not changed" constraint, and passes with the fix -/// reverted. The failure reaching the summary is what the steps above assert. +/// The only assertion behind the "ROCm CLI does not stop the server" promise +/// (README and `rocm serve --help`): a failed smoke test leaves the service +/// record `ready`. It passes with the fix reverted, so it does not check the +/// summary change; the steps above do. The exit code 0 is asserted in the `When` +/// step (`wait_for_exit`). #[then("the launch's service record is left as ready")] async fn assert_service_record_unchanged(world: &mut E2eWorld) { assert_eq!( @@ -3981,9 +3991,12 @@ async fn serve_wait_refused(world: &mut E2eWorld) { ); } -/// The discriminating assertion for the fabricated-launch daemon suppression. -/// Hook hygiene, shared by serve-41 and serve-60: it checks the test seam, not -/// the behaviour either scenario is about. +/// Hook hygiene, shared by the `@id:serve-oom-launch-memory-guidance` and +/// `@id:serve-smoke-failure-summary` scenarios: it records that no daemon was +/// started, and it does NOT discriminate the `if !e2e_simulate_oom_launch()` +/// guard in `serve_cmd.rs`. `ensure_background_helper_running_with` returns early +/// unless a watcher is enabled, and the isolated config these scenarios run in +/// enables none, so removing the guard leaves both scenarios green. /// /// `simulate_oom_managed_launch` writes a record (`starting`, or `ready` in its /// `ready` mode) for a process that never existed, so @@ -3992,12 +4005,11 @@ async fn serve_wait_refused(world: &mut E2eWorld) { /// scenario's teardown stops it. Observed in the wild as daemons outliving their /// e2e checkout by days. /// -/// Without this step the suppression was untested: deleting the -/// `if !e2e_simulate_oom_launch()` guard in `serve_cmd.rs` left serve-41 green, -/// because every other assertion here is about the summary text. Both artefacts -/// the autostart path can leave are checked, since the daemon publishes its -/// runtime-state only after it is up — a spawn caught in the window before that -/// shows up as a claim and nothing else. +/// The guard would only matter in a config with a watcher enabled, which no +/// scenario here sets up; this step would catch a daemon in that case. Both +/// artefacts the autostart path can leave are checked, since the daemon +/// publishes its runtime-state only after it is up — a spawn caught in the +/// window before that shows up as a claim and nothing else. #[then("no background automation daemon was left behind")] async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { let root = world