From e38b064ee2560728f085b6f0a6d3877ad7f81742 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:27:15 +0000 Subject: [PATCH 1/5] Initial plan Signed-off-by: Jussi Elo From 62b2cd4d3d664362cf9a57ab82062d3a5aa0cd66 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:42:01 +0000 Subject: [PATCH 2/5] test(e2e): capture Lemonade backend identity on serve failures Signed-off-by: GitHub Co-authored-by: jussielo-amd <194791304+jussielo-amd@users.noreply.github.com> Signed-off-by: Jussi Elo --- Cargo.lock | 1 + docs/testing.md | 32 ++ tests/e2e-cucumber/Cargo.toml | 3 +- tests/e2e-cucumber/expectations.toml | 100 ++--- tests/e2e-cucumber/src/expectation.rs | 33 ++ .../e2e-cucumber/src/lemonade_diagnostics.rs | 403 ++++++++++++++++++ tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/serve_log.rs | 4 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 51 ++- 9 files changed, 552 insertions(+), 76 deletions(-) create mode 100644 tests/e2e-cucumber/src/lemonade_diagnostics.rs diff --git a/Cargo.lock b/Cargo.lock index b9c13d8376..8e63220c4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "sha2", "tempfile", "tokio", "toml", diff --git a/docs/testing.md b/docs/testing.md index 525cb9fb99..3f3e6c14e8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -815,6 +815,38 @@ Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"); check that line before relying on this coverage. +### Lemonade managed-serve failure evidence + +On a failed serve, the shared serve steps preserve the full service log under +`service-logs/` in the E2E results artifact before stopping the service. Lemonade +failures also include the installed runtime manifest's version/backend, configured +`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints +and `version.txt`, +and each candidate's `--version` and `--help` output. Probes are read-only, +time-bounded (5 seconds each, 20 seconds total), and output-capped at 64 KiB each. +A candidate or configured build pin is **not proof** of which binary the router +launched; a failed probe is retained as evidence, not treated as an identity. +Probes inherit the harness environment rather than reconstructing the engine's +managed library paths, so a probe loader error alone does not establish why the +serve failed. + +[Issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the distinct +Ubuntu signatures: `--no-mmap` rejection, backend-ready/router-unavailable, and +backend download failure. Do not equate the backend's ready line on port 8001 +with a working model on port 11435, or infer a shutdown without lifecycle evidence. +The existing hardware scenarios `@id:bench-load-real-serve`, +`@id:chat-tool-definitions-accepted`, `@id:chat-end-to-end-local-model`, +`@id:serve-lemonade-inference`, and `@id:serve-stop-leaves-no-background-helper` +exercise these paths on the Strix Halo Ubuntu release/nightly lanes. This change +only adds harness diagnostics and corrects xfail attribution; it does not change +the CLI's asynchronous managed-launch contract or claim to fix a backend. + +Exercise the diagnostics and expectation scoping without a GPU: + +```bash +cargo test -p e2e-cucumber --lib lemonade +``` + ## Runtime Selection And Activation List registered managed or read-only runtimes and their exact side-by-side keys: diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 719f32ac35..1c9f9f3b72 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -61,8 +61,9 @@ rand.workspace = true reqwest = { version = "0.13", features = ["json"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tempfile = "3" -tokio.workspace = true +tokio = { workspace = true, features = ["process", "io-util"] } toml = "1.1" # `ServeDir` backs the install scenario's loopback download server (see # `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing, diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index fec5e0154f..b2d4ec4524 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -43,60 +43,44 @@ when = {} bug = "EAI-7219" reason = "Engines do not report a consistent resolved model name for a short name." -# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: -# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run -# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails -# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry -# `flaky = true` — either outcome is tolerated while the bug stays open. The flag -# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added -# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade -# build a given run used cannot currently be read back from this lane, so an -# XPASS streak here is not by itself evidence that the bug is fixed. +# ROCm/rocm-cli#897 tracks multiple Strix Halo Ubuntu Lemonade serve signatures: +# the child rejects --no-mmap (41984921053), backend readiness on :8001 without +# the model appearing on :11435 (41984503055), and backend download failure +# (41986772728). A backend ready line does not prove a later shutdown. +# EAI-7423 described only the earlier ready-then-unavailable symptom. +# Retain flaky=true because historical runs also passed intermittently; do not +# infer a fix from a single XPASS. Failed serves now capture candidate executable +# --version/--help and configured pins separately, before stopping the service. # -# On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve -# loads the model and reaches ready on :8001, then is shut down ~0.08s later, so -# the endpoint never becomes reachable (EAI-7423, root-caused on the box -# 2026-07-15 — NOT the earlier "backend install hides the plan line" theory, which -# was disproven: the `engine:` line IS emitted). os=linux: on native Windows this -# scenario PASSES (run 29338475133 Strix-Windows), so it must NOT be xfail'd there. +# os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade-native LINUX host the served model's endpoint drops before -# inference: the lemonade managed serve reaches ready then is shut down ~0.08s -# later (EAI-7423, root-caused on the box 2026-07-15 — not a Vulkan hang). -# os=linux: passes on native Windows (run 29338475133). +# The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade LINUX host this scenario does a real lemonade serve, which reaches -# ready then shuts down immediately (EAI-7423, see serve-lemonade-inference). -# os=linux: on native Windows lemonade serve works, so this PASSES there (verified -# run 29338475133 Strix-Windows XPASS) and must not be xfail'd. +# The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is -# shut down immediately, so inference never happens (root-caused on the box -# 2026-07-15; previously mis-attributed to the EAI-7052 Vulkan hang). os=linux: -# native Windows lemonade serve works (run 29338475133 Strix-Windows XPASS'd -# this), so scope the xfail to Linux. --- +# Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -107,16 +91,11 @@ bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# On a lemonade LINUX host this needs a live lemonade serve, which reaches ready -# then shuts down immediately (EAI-7423). os=linux: passes on native Windows -# lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the -# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a -# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -125,15 +104,12 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade -# serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to -# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not -# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# path works. Only the real Lemonade Linux serve stays xfail under #897; +# the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -144,15 +120,7 @@ when = {} bug = "EAI-7384" reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-provisioning." -# The GPU-backed bench scenario benchmarks a really served model, so on a -# lemonade LINUX host it inherits the same managed-serve bug as the sibling -# serve/chat rows below `serve-default-engine-working-endpoint`: the serve -# reaches ready then shuts down immediately, so this scenario's `a model is -# being served on GPU` precondition never yields a live endpoint. Scoped -# identically — os=linux only, because native Windows lemonade serve works, and -# therock_family=gfx* so it applies only on a real GPU host. `flaky = true` -# matches those rows too: which of them pass varies run to run on that lane, and -# a deterministic-XPASS row would fail the lane on unrelated PRs. +# Bench's real-serve precondition is blocked by #897 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -160,7 +128,15 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." +serve_timeout_secs = 90 +flaky = true + +# serve-40 has the same serve precondition; this is not an automation-helper bug. +[["serve-stop-leaves-no-background-helper"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCm/rocm-cli#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index df78c81db4..b930a1cc3e 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -826,6 +826,39 @@ mod tests { } } + #[test] + fn lemonade_serve_failures_track_897_without_xfailing_mock_or_windows() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + ] { + let scenario = decl(&[&format!("id:{id}")]); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCm/rocm-cli#897" + ), + "{id}" + ); + assert_eq!( + matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + Some(90), + "{id}", + ); + for platform in ["mock", "strix-windows"] { + assert_eq!( + resolve(&scenario, &cap(platform), &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {platform}", + ); + } + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs new file mode 100644 index 0000000000..2b5e38f9e0 --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -0,0 +1,403 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read-only backend evidence for a failed Lemonade serve, before teardown. + +use sha2::{Digest as _, Sha256}; +use std::fmt::Write as _; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::Duration; +use tokio::io::{AsyncRead, AsyncReadExt as _}; + +use crate::serve_log::read_clamped; + +const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CANDIDATES: usize = 8; +const TOTAL_PROBE_TIMEOUT: Duration = Duration::from_secs(20); + +/// Record the installed version and pins separately from executable build output: +/// a pin describes what should be installed, not what a runner actually cached. +/// +/// Every candidate is labelled as such; finding a binary does not prove that the +/// router launched it. No install, download, model load, or cache repair is done. +pub async fn backend_evidence(manifest_path: &Path) -> String { + let mut report = String::new(); + let _ = writeln!(report, "manifest: {}", manifest_path.display()); + let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return format!("{report}unavailable: {error}"), + }; + for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { + let _ = writeln!( + report, + "{key}: {}", + manifest.get(key).unwrap_or(&serde_json::Value::Null) + ); + } + let Some(runtime) = manifest + .get("runtime_dir") + .and_then(serde_json::Value::as_str) + else { + return format!("{report}backend inventory unavailable: no runtime_dir"); + }; + let runtime = Path::new(runtime); + let pins = runtime.join("resources/backend_versions.json"); + let _ = writeln!(report, "configured pins (not executable identity):"); + match read_clamped(&pins, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); + } + Err(error) => { + let _ = writeln!(report, "{}: {error}", pins.display()); + } + } + let candidates = backend_candidates(runtime); + let _ = writeln!( + report, + "probes inherit the harness environment, not the engine's managed library environment; \ + a probe loader failure does not establish a serving failure" + ); + if candidates.is_empty() { + let _ = writeln!( + report, + "no llama-server candidates under bin/llamacpp (backend download may not have completed)" + ); + } + let deadline = tokio::time::Instant::now() + TOTAL_PROBE_TIMEOUT; + for server in candidates.iter().take(MAX_CANDIDATES) { + let _ = writeln!( + report, + "candidate (not proof of selection): {}", + server.display() + ); + match fingerprint(server) { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + } + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining probes skipped: 20s evidence budget exhausted" + ); + return report; + } + let output = probe(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + if candidates.len() > MAX_CANDIDATES { + let _ = writeln!( + report, + "backend inventory truncated after {MAX_CANDIDATES} candidates" + ); + } + report +} + +fn backend_candidates(runtime: &Path) -> Vec { + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut candidates = Vec::new(); + for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { + let dir = runtime.join("bin/llamacpp").join(backend); + let direct = dir.join(binary); + if direct.is_file() { + candidates.push(direct); + } + // Lemonade archives also unpack one level below the backend directory. + if let Ok(entries) = std::fs::read_dir(dir) { + let mut nested: Vec<_> = entries + .filter_map(Result::ok) + .map(|entry| entry.path().join(binary)) + .filter(|path| path.is_file()) + .collect(); + nested.sort(); + candidates.extend(nested); + } + } + candidates +} + +fn fingerprint(server: &Path) -> std::io::Result { + let file = std::fs::File::open(server)?; + // A real server is much smaller; do not hash an arbitrarily large corrupt + // cache entry while reporting a failure. Never label a partial hash as SHA256. + let limit = 512 * 1024 * 1024; + let mut hasher = Sha256::new(); + let count = std::io::copy(&mut file.take(limit + 1), &mut hasher)?; + if count > limit { + return Err(std::io::Error::other( + "executable exceeds 512 MiB hash limit", + )); + } + Ok(format!("{:x}", hasher.finalize())) +} + +async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { + let mut command = tokio::process::Command::new(server); + command + .arg(flag) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return format!("probe could not start: {error}"), + }; + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let mut out = Vec::new(); + let mut err = Vec::new(); + let status = if let Ok((status, stdout_result, stderr_result)) = + tokio::time::timeout(timeout, async { + tokio::join!( + child.wait(), + drain_output(stdout, &mut out), + drain_output(stderr, &mut err), + ) + }) + .await + { + format!( + "{}\nstdout capture: {stdout_result:?}; stderr capture: {stderr_result:?}", + status.map_or_else(|error| error.to_string(), |status| status.to_string()) + ) + } else { + // kill_on_drop also covers cancellation of the whole reporter. + // Explicitly wait here so a normal timeout leaves no zombie child. + let result = child.kill().await; + format!( + "probe timed out after {}s; kill/reap: {result:?}", + timeout.as_secs_f64() + ) + }; + format!( + "{status}\nstdout:\n{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out), + String::from_utf8_lossy(&err), + if out.len() == MAX_EVIDENCE_BYTES as usize / 2 + || err.len() == MAX_EVIDENCE_BYTES as usize / 2 + { + "\n" + } else { + "" + } + ) +} + +async fn drain_output( + mut reader: impl AsyncRead + Unpin, + retained: &mut Vec, +) -> std::io::Result<()> { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + let mut buffer = [0; 4096]; + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + // Continue draining both pipes after the cap, without retaining more + // bytes or writing them to disk. A noisy child must not block on output. + let keep = count.min(limit.saturating_sub(retained.len())); + retained.extend_from_slice(&buffer[..keep]); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn executable_identity_does_not_depend_on_the_dynamic_loader() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("not-executable"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut wanted = Vec::new(); + for relative in ["rocm-stable", "rocm-nightly/llama-b9752", "vulkan/build"] { + let server = root.path().join("bin/llamacpp").join(relative).join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, "").unwrap(); + wanted.push(server); + } + let cpu = root.path().join("bin/llamacpp/cpu").join(binary); + std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); + std::fs::write(cpu, "").unwrap(); + assert_eq!(backend_candidates(root.path()), wanted); + } + + #[tokio::test] + async fn missing_manifest_is_reported_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let report = backend_evidence(&root.path().join("missing.json")).await; + assert!(report.contains("unavailable:"), "{report}"); + } + + #[tokio::test] + async fn failed_download_keeps_pins_but_does_not_invent_an_installed_build() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + std::fs::create_dir_all(runtime.join("resources")).unwrap(); + std::fs::write( + runtime.join("resources/backend_versions.json"), + r#"{"llamacpp":{"rocm-stable":"b9752"}}"#, + ) + .unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!( + report.contains("configured pins (not executable identity):"), + "{report}" + ); + assert!(report.contains("\"rocm-stable\":\"b9752\""), "{report}"); + assert!(report.contains("no llama-server candidates"), "{report}"); + } + + #[tokio::test] + async fn probe_start_error_is_evidence_not_a_second_failure() { + let root = tempfile::tempdir().unwrap(); + let report = probe(&root.path().join("missing"), "--version", PROBE_TIMEOUT).await; + assert!(report.contains("probe could not start:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn inventory_probes_the_cached_executable_not_a_path_lookup() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + let dir = runtime.join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(&dir).unwrap(); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "\ +#!/bin/sh +case \"$1\" in + --version) echo 'version: 9752 (fixture-build)' ;; + --help) echo '--mmap, --no-mmap'; echo 'fixture help stderr' >&2; exit 2 ;; + *) exit 99 ;; +esac +", + ) + .unwrap(); + std::fs::write(dir.join("version.txt"), "cached-build").unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); + assert!(report.contains("version.txt: cached-build"), "{report}"); + assert!(report.contains("--help:\nexit status: 2"), "{report}"); + assert!(report.contains("--mmap, --no-mmap"), "{report}"); + assert!(report.contains("fixture help stderr"), "{report}"); + assert!( + server.is_file(), + "diagnostics must not repair or remove the cache" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn probe_timeout_is_bounded() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + // exec keeps the sleeper in the child's PID, so cancellation kills it + // rather than leaving a grandchild behind. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\necho $$\nexec /bin/sleep 30\n", + ) + .unwrap(); + let start = std::time::Instant::now(); + let report = probe(&server, "--help", Duration::from_millis(50)).await; + assert!(report.contains("probe timed out after 0.05s"), "{report}"); + assert!(start.elapsed() < Duration::from_secs(5)); + let pid = report + .split("stdout:\n") + .nth(1) + .unwrap() + .lines() + .next() + .unwrap() + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn noisy_probe_output_is_capped() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 100000 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!( + report.ends_with(""), + "{report}" + ); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 200); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 6d16d6c4dd..f56a347bfa 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -9,6 +9,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_diagnostics; pub mod loopback_http; pub mod mock_server; pub mod model_id; diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index bf0ff9d8c1..c90581d26d 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -120,14 +120,14 @@ fn slugify(name: &str) -> String { /// /// The log may still be being written, so `len` is a snapshot: the head and tail /// are always real, and only the elided count can be slightly stale. -fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { +pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; let mut file = std::fs::File::open(path)?; let len = file.metadata()?.len(); if len <= max { let mut body = Vec::new(); - file.read_to_end(&mut body)?; + file.take(max).read_to_end(&mut body)?; return Ok((body, String::new())); } // `len > max >= 2 * half`, so the two halves can never overlap. diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 521629c8ed..0cb1e95836 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -138,8 +138,9 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( +async fn serve_failure_evidence( world: &E2eWorld, + engine: &str, headline: &str, device_state: &str, stdout: &str, @@ -154,8 +155,19 @@ fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); + let backend_evidence = if engine == "lemonade" { + e2e_cucumber::lemonade_diagnostics::backend_evidence( + &world + .app_paths() + .engine_manifests_dir("lemonade") + .join("runtime.json"), + ) + .await + } else { + "not a Lemonade serve".to_owned() + }; let stop_status = stop_scenario_services(world); - serve_attempt_report(&ServeAttempt { + let report = serve_attempt_report(&ServeAttempt { headline, device_state, stdout, @@ -163,7 +175,8 @@ fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, - }) + }); + format!("{report}\n--- Lemonade backend evidence ---\n{backend_evidence}") } /// Launch a managed serve and wait until the endpoint really serves it, failing @@ -201,7 +214,17 @@ async fn serve_and_wait( }; panic!( "{}", - serve_failure_evidence(world, &headline, &device_state, &stdout, &stderr) + serve_failure_evidence( + world, + args.windows(2) + .find(|pair| pair[0] == "--engine") + .map_or("", |pair| pair[1]), + &headline, + &device_state, + &stdout, + &stderr, + ) + .await ); } @@ -608,13 +631,17 @@ async fn serve_gpu_model(world: &mut E2eWorld) { // outcome is quoted: a stop that found no record (the launch had not yet // written one) or failed means the next attempt did NOT get a clean // device, and the failure says so instead of looking like a broken serve. - diagnostics.push(serve_failure_evidence( - world, - &format!("attempt {attempt} of {attempts} (rc={rc})"), - &device_state, - &stdout, - &stderr, - )); + diagnostics.push( + serve_failure_evidence( + world, + engine, + &format!("attempt {attempt} of {attempts} (rc={rc})"), + &device_state, + &stdout, + &stderr, + ) + .await, + ); if attempt == attempts { break; } @@ -981,11 +1008,13 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { "{}", serve_failure_evidence( world, + selected_engine(world.cli_output.as_deref().unwrap_or_default()), &headline, &device_state, world.cli_output.as_deref().unwrap_or_default(), world.cli_stderr.as_deref().unwrap_or_default(), ) + .await ); } } From 55220d7eb4c30a335a056d77e10ea7bc4065233c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:47:15 +0000 Subject: [PATCH 3/5] test(e2e): keep async serve evidence futures Send Signed-off-by: GitHub Co-authored-by: jussielo-amd <194791304+jussielo-amd@users.noreply.github.com> Signed-off-by: Jussi Elo --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 0cb1e95836..746fca51b9 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -139,7 +139,7 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. async fn serve_failure_evidence( - world: &E2eWorld, + world: &mut E2eWorld, engine: &str, headline: &str, device_state: &str, @@ -1004,15 +1004,17 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { &ready_substr, timeout_secs, ); + let stdout = world.cli_output.clone().unwrap_or_default(); + let stderr = world.cli_stderr.clone().unwrap_or_default(); panic!( "{}", serve_failure_evidence( world, - selected_engine(world.cli_output.as_deref().unwrap_or_default()), + selected_engine(&stdout), &headline, &device_state, - world.cli_output.as_deref().unwrap_or_default(), - world.cli_stderr.as_deref().unwrap_or_default(), + &stdout, + &stderr, ) .await ); From 1e5589450ff503adc1e28ffea0a228b213111f55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 5 Nov 2026 09:17:39 +0000 Subject: [PATCH 4/5] test(e2e): harden Lemonade failure evidence bounds Signed-off-by: GitHub Co-authored-by: jussielo-amd <194791304+jussielo-amd@users.noreply.github.com> Signed-off-by: Jussi Elo --- docs/testing.md | 28 +- tests/e2e-cucumber/expectations.toml | 16 +- .../features/model_serving.feature | 2 +- tests/e2e-cucumber/src/expectation.rs | 2 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 295 +++++++++++++++--- tests/e2e-cucumber/src/serve_log.rs | 44 ++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 32 +- 7 files changed, 339 insertions(+), 80 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3f3e6c14e8..1545157004 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -822,24 +822,28 @@ On a failed serve, the shared serve steps preserve the full service log under failures also include the installed runtime manifest's version/backend, configured `backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints and `version.txt`, -and each candidate's `--version` and `--help` output. Probes are read-only, -time-bounded (5 seconds each, 20 seconds total), and output-capped at 64 KiB each. +and each candidate's `--version` and `--help` output. At most eight candidates +are hashed and probed, with two read-only probes per candidate. Each probe retains +up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are +marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw +output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). +Each probe has a five-second timeout. Hashing and probes share a 20-second budget +after inventory discovery; hashing runs off the async executor, rejects files +over 32 MiB, and checks its deadline between 64 KiB reads. Filesystem operations +and process cleanup are not preemptible, so this is not a hard wall-clock limit +on the entire report. A candidate or configured build pin is **not proof** of which binary the router launched; a failed probe is retained as evidence, not treated as an identity. Probes inherit the harness environment rather than reconstructing the engine's managed library paths, so a probe loader error alone does not establish why the serve failed. -[Issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the distinct -Ubuntu signatures: `--no-mmap` rejection, backend-ready/router-unavailable, and -backend download failure. Do not equate the backend's ready line on port 8001 -with a working model on port 11435, or infer a shutdown without lifecycle evidence. -The existing hardware scenarios `@id:bench-load-real-serve`, -`@id:chat-tool-definitions-accepted`, `@id:chat-end-to-end-local-model`, -`@id:serve-lemonade-inference`, and `@id:serve-stop-leaves-no-background-helper` -exercise these paths on the Strix Halo Ubuntu release/nightly lanes. This change -only adds harness diagnostics and corrects xfail attribution; it does not change -the CLI's asynchronous managed-launch contract or claim to fix a backend. +Use the evidence to distinguish backend startup, download, and router readiness +failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the +Strix Halo Ubuntu investigation. A backend's ready line does not establish that +the router serves the requested model, and a readiness timeout alone does not +establish a shutdown. Diagnostics do not change the CLI's asynchronous +managed-launch contract or imply a backend fix. Exercise the diagnostics and expectation scoping without a GPU: diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index b2d4ec4524..fd979b7c30 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -55,7 +55,7 @@ reason = "Engines do not report a consistent resolved model name for a short nam # os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -63,7 +63,7 @@ flaky = true # The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -71,7 +71,7 @@ flaky = true # The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -79,7 +79,7 @@ flaky = true # Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -94,7 +94,7 @@ serve_timeout_secs = 90 # Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -108,7 +108,7 @@ flaky = true # the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -128,7 +128,7 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -136,7 +136,7 @@ flaky = true # serve-40 has the same serve precondition; this is not an automation-helper bug. [["serve-stop-leaves-no-background-helper"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "ROCm/rocm-cli#897" +bug = "#897" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 539644f521..1dcf18217f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -75,7 +75,7 @@ Feature: Model serving # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found # under bin/llamacpp// — unlike the short-recipe-name path above, whose # managed-lemonade fallback would mask the identical failure behind the - # unrelated EAI-7423 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as + # #897 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index b930a1cc3e..60c58f495f 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -840,7 +840,7 @@ mod tests { assert!( matches!( resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), - Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCm/rocm-cli#897" + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "#897" ), "{id}" ); diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 2b5e38f9e0..0c5ace2a81 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -9,7 +9,7 @@ use std::fmt::Write as _; use std::io::Read as _; use std::path::{Path, PathBuf}; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncReadExt as _}; use crate::serve_log::read_clamped; @@ -17,7 +17,8 @@ use crate::serve_log::read_clamped; const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; const PROBE_TIMEOUT: Duration = Duration::from_secs(5); const MAX_CANDIDATES: usize = 8; -const TOTAL_PROBE_TIMEOUT: Duration = Duration::from_secs(20); +const TOTAL_EVIDENCE_BUDGET: Duration = Duration::from_secs(20); +const MAX_HASH_BYTES: u64 = 32 * 1024 * 1024; /// Record the installed version and pins separately from executable build output: /// a pin describes what should be installed, not what a runner actually cached. @@ -25,6 +26,10 @@ const TOTAL_PROBE_TIMEOUT: Duration = Duration::from_secs(20); /// Every candidate is labelled as such; finding a binary does not prove that the /// router launched it. No install, download, model load, or cache repair is done. pub async fn backend_evidence(manifest_path: &Path) -> String { + backend_evidence_with_budget(manifest_path, TOTAL_EVIDENCE_BUDGET).await +} + +async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> String { let mut report = String::new(); let _ = writeln!(report, "manifest: {}", manifest_path.display()); let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { @@ -69,14 +74,26 @@ pub async fn backend_evidence(manifest_path: &Path) -> String { "no llama-server candidates under bin/llamacpp (backend download may not have completed)" ); } - let deadline = tokio::time::Instant::now() + TOTAL_PROBE_TIMEOUT; + let deadline = Instant::now() + budget; for server in candidates.iter().take(MAX_CANDIDATES) { + if Instant::now() >= deadline { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } let _ = writeln!( report, "candidate (not proof of selection): {}", server.display() ); - match fingerprint(server) { + let path = server.clone(); + match tokio::task::spawn_blocking(move || fingerprint(&path, deadline)) + .await + .map_err(std::io::Error::other) + .and_then(|result| result) + { Ok(hash) => { let _ = writeln!(report, "sha256: {hash}"); } @@ -96,11 +113,11 @@ pub async fn backend_evidence(manifest_path: &Path) -> String { } } for flag in ["--version", "--help"] { - let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + let remaining = deadline.saturating_duration_since(Instant::now()); if remaining.is_zero() { let _ = writeln!( report, - "remaining probes skipped: 20s evidence budget exhausted" + "remaining evidence skipped: evidence budget exhausted" ); return report; } @@ -144,21 +161,58 @@ fn backend_candidates(runtime: &Path) -> Vec { candidates } -fn fingerprint(server: &Path) -> std::io::Result { - let file = std::fs::File::open(server)?; +fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + let mut file = std::fs::File::open(server)?; // A real server is much smaller; do not hash an arbitrarily large corrupt // cache entry while reporting a failure. Never label a partial hash as SHA256. - let limit = 512 * 1024 * 1024; - let mut hasher = Sha256::new(); - let count = std::io::copy(&mut file.take(limit + 1), &mut hasher)?; - if count > limit { + if file.metadata()?.len() > MAX_HASH_BYTES { return Err(std::io::Error::other( - "executable exceeds 512 MiB hash limit", + "executable exceeds 32 MiB hash limit", )); } + let mut hasher = Sha256::new(); + let mut count = 0; + let mut buffer = vec![0; 64 * 1024]; + loop { + // Check inside the worker, not just around its JoinHandle: timing out a + // spawn_blocking await alone would leave the hashing thread running. + if Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "hash evidence budget exhausted", + )); + } + let read = file.read(&mut buffer)?; + count += read as u64; + if count > MAX_HASH_BYTES { + return Err(std::io::Error::other( + "executable exceeds 32 MiB hash limit", + )); + } + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } Ok(format!("{:x}", hasher.finalize())) } +#[derive(Default)] +struct CapturedOutput { + bytes: Vec, + truncated: bool, +} + +impl CapturedOutput { + fn truncation_note(&self, stream: &str) -> String { + if self.truncated { + format!("\n<{stream} truncated at 32 KiB>") + } else { + String::new() + } + } +} + async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { let mut command = tokio::process::Command::new(server); command @@ -173,8 +227,8 @@ async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { }; let stdout = child.stdout.take().expect("piped stdout"); let stderr = child.stderr.take().expect("piped stderr"); - let mut out = Vec::new(); - let mut err = Vec::new(); + let mut out = CapturedOutput::default(); + let mut err = CapturedOutput::default(); let status = if let Ok((status, stdout_result, stderr_result)) = tokio::time::timeout(timeout, async { tokio::join!( @@ -199,22 +253,17 @@ async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { ) }; format!( - "{status}\nstdout:\n{}\nstderr:\n{}{}", - String::from_utf8_lossy(&out), - String::from_utf8_lossy(&err), - if out.len() == MAX_EVIDENCE_BYTES as usize / 2 - || err.len() == MAX_EVIDENCE_BYTES as usize / 2 - { - "\n" - } else { - "" - } + "{status}\nstdout:\n{}{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out.bytes), + out.truncation_note("stdout"), + String::from_utf8_lossy(&err.bytes), + err.truncation_note("stderr"), ) } async fn drain_output( mut reader: impl AsyncRead + Unpin, - retained: &mut Vec, + retained: &mut CapturedOutput, ) -> std::io::Result<()> { let limit = MAX_EVIDENCE_BYTES as usize / 2; let mut buffer = [0; 4096]; @@ -225,8 +274,9 @@ async fn drain_output( } // Continue draining both pipes after the cap, without retaining more // bytes or writing them to disk. A noisy child must not block on output. - let keep = count.min(limit.saturating_sub(retained.len())); - retained.extend_from_slice(&buffer[..keep]); + let keep = count.min(limit.saturating_sub(retained.bytes.len())); + retained.bytes.extend_from_slice(&buffer[..keep]); + retained.truncated |= keep < count; } } @@ -234,17 +284,59 @@ async fn drain_output( mod tests { use super::*; + fn write_manifest(runtime: &Path) -> PathBuf { + let manifest = runtime.join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + manifest + } + #[test] fn executable_identity_does_not_depend_on_the_dynamic_loader() { let root = tempfile::tempdir().unwrap(); let server = root.path().join("not-executable"); std::fs::write(&server, b"abc").unwrap(); assert_eq!( - fingerprint(&server).unwrap(), + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET).unwrap(), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } + #[test] + fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + let file = std::fs::File::create(&server).unwrap(); + file.set_len(MAX_HASH_BYTES).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap() + .len(), + 64 + ); + file.set_len(MAX_HASH_BYTES + 1).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap_err() + .to_string(), + "executable exceeds 32 MiB hash limit" + ); + } + + #[test] + fn hash_worker_obeys_its_own_deadline() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now()).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + #[test] fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { let root = tempfile::tempdir().unwrap(); @@ -266,6 +358,118 @@ mod tests { assert_eq!(backend_candidates(root.path()), wanted); } + #[tokio::test] + async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + for i in 0..=MAX_CANDIDATES { + let server = root + .path() + .join(format!("bin/llamacpp/rocm-stable/{i:02}/{binary}")); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + let report = backend_evidence(&write_manifest(root.path())).await; + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + assert_eq!(report.matches("sha256:").count(), MAX_CANDIDATES); + assert_eq!(report.matches("--help:").count(), MAX_CANDIDATES); + assert!(report.contains("backend inventory truncated after 8 candidates")); + let skipped = root + .path() + .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); + assert!(!report.contains(&skipped.display().to_string()), "{report}"); + } + + #[tokio::test] + async fn exhausted_budget_skips_hashing_and_probes() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::ZERO).await; + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + for skipped in ["sha256", "--version:", "--help:", "candidate ("] { + assert!(!report.contains(skipped), "{report}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("bin/llamacpp/rocm-stable/llama-server"); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nexec /bin/sleep 30\n", + ) + .unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::from_secs(1)) + .await; + assert!(report.contains("--version:\nprobe timed out"), "{report}"); + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + assert!(!report.contains("--help:"), "{report}"); + } + + #[tokio::test] + async fn stream_capture_marks_only_bytes_actually_discarded() { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + for length in [0, limit - 1, limit, limit + 1] { + let bytes = vec![b'x'; length]; + let mut captured = CapturedOutput::default(); + drain_output(bytes.as_slice(), &mut captured).await.unwrap(); + assert_eq!(captured.bytes, vec![b'x'; length.min(limit)]); + assert_eq!(captured.truncated, length > limit); + assert_eq!( + captured.truncation_note("stdout"), + if length > limit { + "\n" + } else { + "" + } + ); + } + } + + #[tokio::test] + async fn stream_truncation_survives_timeout_before_eof() { + use tokio::io::AsyncWriteExt as _; + let length = MAX_EVIDENCE_BYTES as usize / 2 + 1; + let (reader, mut writer) = tokio::io::duplex(length); + writer.write_all(&vec![b'x'; length]).await.unwrap(); + let mut captured = CapturedOutput::default(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + drain_output(reader, &mut captured) + ) + .await + .is_err() + ); + assert_eq!(captured.bytes.len(), length - 1); + assert_eq!( + captured.truncation_note("stderr"), + "\n" + ); + } + #[tokio::test] async fn missing_manifest_is_reported_without_panicking() { let root = tempfile::tempdir().unwrap(); @@ -368,8 +572,8 @@ esac ) .unwrap(); let start = std::time::Instant::now(); - let report = probe(&server, "--help", Duration::from_millis(50)).await; - assert!(report.contains("probe timed out after 0.05s"), "{report}"); + let report = probe(&server, "--help", Duration::from_secs(1)).await; + assert!(report.contains("probe timed out after 1s"), "{report}"); assert!(start.elapsed() < Duration::from_secs(5)); let pid = report .split("stdout:\n") @@ -383,6 +587,23 @@ esac assert!(!rocm_core::process_is_running(pid), "{report}"); } + #[cfg(unix)] + #[tokio::test] + async fn exact_stream_limit_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 32768 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains("exit status: 0"), "{report}"); + assert_eq!(report.bytes().filter(|byte| *byte == 0).count(), 32768); + assert!(!report.contains("capped"), "{report}"); + assert!(!report.contains("truncated"), "{report}"); + } + #[cfg(unix)] #[tokio::test] async fn noisy_probe_output_is_capped() { @@ -390,14 +611,16 @@ esac let server = root.path().join("llama-server"); rocm_core::fake_executable::write_fake_executable( &server, - "#!/bin/sh\nhead -c 100000 /dev/zero\n", + "#!/bin/sh\nhead -c 100000 /dev/zero\nhead -c 100000 /dev/zero >&2\n", ) .unwrap(); let report = probe(&server, "--help", PROBE_TIMEOUT).await; - assert!( - report.ends_with(""), - "{report}" + assert!(report.contains(""), "{report}"); + assert!(report.ends_with(""), "{report}"); + assert_eq!( + report.bytes().filter(|byte| *byte == 0).count(), + MAX_EVIDENCE_BYTES as usize ); - assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 200); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 300); } } diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index c90581d26d..e0f82d8f6a 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -55,6 +55,16 @@ fn parse_log_path(serve_stdout: &str) -> Option<&str> { .filter(|path| !path.is_empty()) } +/// The selected engine, if the serve reached the point of printing its plan. +#[must_use] +pub fn reported_engine(serve_stdout: &str) -> Option<&str> { + serve_stdout + .lines() + .find_map(|line| line.trim().strip_prefix("engine:")) + .map(str::trim) + .filter(|engine| !engine.is_empty()) +} + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -230,6 +240,8 @@ pub struct ServeAttempt<'a> { /// failed, means the engine may still hold the port and the device — which /// changes how every later failure in the run should be read. pub stop_status: &'a str, + /// Only Lemonade attempts carry the optional backend inventory. + pub backend_evidence: Option<&'a str>, } /// Render one serve attempt's evidence as a failure message. @@ -241,7 +253,7 @@ pub struct ServeAttempt<'a> { #[must_use] pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { use crate::section; - format!( + let mut report = format!( "{}\n{}\n{}\n{}\n{}\n{}\n{}", attempt.headline, section("device state", attempt.device_state), @@ -250,14 +262,19 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { section("service log (tail)", attempt.log_tail), section("service log (full)", attempt.archived_log), section("stop", attempt.stop_status), - ) + ); + if let Some(evidence) = attempt.backend_evidence { + report.push_str("\n--- Lemonade backend evidence ---\n"); + report.push_str(evidence); + } + report } #[cfg(test)] mod tests { use super::{ MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - serve_attempt_report, service_log_tail, slugify, tail_lines, + reported_engine, serve_attempt_report, service_log_tail, slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -270,6 +287,15 @@ managed service launched readiness: starting "; + #[test] + fn engine_lookup_is_safe_for_missing_or_incomplete_serve_plans() { + assert_eq!(reported_engine(SERVE_PLAN), Some("vllm")); + assert_eq!(reported_engine(" engine: lemonade\n"), Some("lemonade")); + for output in ["", "serve failed", " engine: \n"] { + assert_eq!(reported_engine(output), None); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( @@ -502,7 +528,7 @@ managed service launched /// full log went — none of which the plain readiness timeout reported. #[test] fn a_stalled_serve_report_carries_every_piece_of_evidence() { - let report = serve_attempt_report(&ServeAttempt { + let mut attempt = ServeAttempt { headline: "`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed` \ exited 0, but the endpoint never served `Qwen3-0.6B` within 600s", device_state: "device state: drained (48000 MiB free of 49000 MiB, floor 44100 MiB)", @@ -511,7 +537,15 @@ managed service launched log_tail: "load: failed to load model", archived_log: "archived to service-logs/scenario--svc.log", stop_status: "lemonade-qwen: stopped", - }); + backend_evidence: None, + }; + let report = serve_attempt_report(&attempt); + assert!(!report.contains("Lemonade backend evidence"), "{report}"); + attempt.backend_evidence = Some("fixture backend identity"); + assert_eq!( + serve_attempt_report(&attempt), + format!("{report}\n--- Lemonade backend evidence ---\nfixture backend identity") + ); for expected in [ "exited 0", diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 746fca51b9..91e0d10d68 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,7 +13,7 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, + ServeAttempt, archive_service_log, reported_engine, serve_attempt_report, service_log_tail, }; /// Model id of the live managed service the reuse scenarios plant. @@ -156,18 +156,20 @@ async fn serve_failure_evidence( world.current_scenario.as_deref().unwrap_or("scenario"), ); let backend_evidence = if engine == "lemonade" { - e2e_cucumber::lemonade_diagnostics::backend_evidence( - &world - .app_paths() - .engine_manifests_dir("lemonade") - .join("runtime.json"), + Some( + e2e_cucumber::lemonade_diagnostics::backend_evidence( + &world + .app_paths() + .engine_manifests_dir("lemonade") + .join("runtime.json"), + ) + .await, ) - .await } else { - "not a Lemonade serve".to_owned() + None }; let stop_status = stop_scenario_services(world); - let report = serve_attempt_report(&ServeAttempt { + serve_attempt_report(&ServeAttempt { headline, device_state, stdout, @@ -175,8 +177,8 @@ async fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, - }); - format!("{report}\n--- Lemonade backend evidence ---\n{backend_evidence}") + backend_evidence: backend_evidence.as_deref(), + }) } /// Launch a managed serve and wait until the endpoint really serves it, failing @@ -1010,7 +1012,7 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { "{}", serve_failure_evidence( world, - selected_engine(&stdout), + reported_engine(&stdout).unwrap_or(""), &headline, &device_state, &stdout, @@ -2469,11 +2471,7 @@ async fn assert_endpoint_port(world: &mut E2eWorld) { /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { - let Some(engine) = output - .lines() - .find_map(|l| l.trim().strip_prefix("engine:")) - .map(str::trim) - else { + let Some(engine) = reported_engine(output) else { panic!("no 'engine:' line in serve output:\n{output}"); }; engine From fc1a14401efb916d9abb168552ba521fc5a1fb70 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 10:24:24 +0000 Subject: [PATCH 5/5] test(e2e): report unknown serve engine and trim probe noise Read the engine from both `--engine X` and `--engine=X`, fall back to the printed serve plan, and say so in the failure report when no engine can be determined instead of silently skipping the Lemonade backend inventory. Print probe capture and kill/reap results only when they fail, and note on the serve-40 expectation row that xfail plus flaky leaves that assertion without signal until #897 is closed. Signed-off-by: Jussi Elo --- docs/testing.md | 6 ++- tests/e2e-cucumber/expectations.toml | 4 ++ .../e2e-cucumber/src/lemonade_diagnostics.rs | 33 ++++++++---- tests/e2e-cucumber/src/serve_log.rs | 51 ++++++++++++++++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 23 ++++----- 5 files changed, 90 insertions(+), 27 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 1545157004..6ecd44e497 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -818,8 +818,10 @@ relying on this coverage. ### Lemonade managed-serve failure evidence On a failed serve, the shared serve steps preserve the full service log under -`service-logs/` in the E2E results artifact before stopping the service. Lemonade -failures also include the installed runtime manifest's version/backend, configured +`service-logs/` in the E2E results artifact before stopping the service. The engine +is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if +neither names one, the report says `engine unknown: backend evidence skipped`. +Lemonade failures also include the installed runtime manifest's version/backend, configured `backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints and `version.txt`, and each candidate's `--version` and `--help` output. At most eight candidates diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index fd979b7c30..60eefd2520 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -134,6 +134,10 @@ serve_timeout_secs = 90 flaky = true # serve-40 has the same serve precondition; this is not an automation-helper bug. +# xfail + flaky tolerates either outcome, so on this lane the "no background +# helper" assertion carries no signal until #897 is closed: a pass here does not +# show the scenario's own behaviour is healthy, and a failure is not attributed +# to it. Remove the row once #897 is fixed. [["serve-stop-leaves-no-background-helper"]] when = { effective_engine = "lemonade", os = "linux" } bug = "#897" diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 0c5ace2a81..0dfcddd7e7 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -239,18 +239,21 @@ async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { }) .await { - format!( - "{}\nstdout capture: {stdout_result:?}; stderr capture: {stderr_result:?}", - status.map_or_else(|error| error.to_string(), |status| status.to_string()) - ) + let mut status = status.map_or_else(|error| error.to_string(), |status| status.to_string()); + for (stream, result) in [("stdout", stdout_result), ("stderr", stderr_result)] { + if let Err(error) = result { + let _ = write!(status, "\n{stream} capture failed: {error}"); + } + } + status } else { // kill_on_drop also covers cancellation of the whole reporter. // Explicitly wait here so a normal timeout leaves no zombie child. - let result = child.kill().await; - format!( - "probe timed out after {}s; kill/reap: {result:?}", - timeout.as_secs_f64() - ) + let mut status = format!("probe timed out after {}s", timeout.as_secs_f64()); + if let Err(error) = child.kill().await { + let _ = write!(status, "; kill/reap failed: {error}"); + } + status }; format!( "{status}\nstdout:\n{}{}\nstderr:\n{}{}", @@ -550,7 +553,12 @@ esac assert!(report.contains(&server.display().to_string()), "{report}"); assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); assert!(report.contains("version.txt: cached-build"), "{report}"); - assert!(report.contains("--help:\nexit status: 2"), "{report}"); + assert!( + report.contains("--help:\nexit status: 2\nstdout:"), + "{report}" + ); + assert!(!report.contains("capture"), "{report}"); + assert!(!report.contains("Ok("), "{report}"); assert!(report.contains("--mmap, --no-mmap"), "{report}"); assert!(report.contains("fixture help stderr"), "{report}"); assert!( @@ -573,7 +581,10 @@ esac .unwrap(); let start = std::time::Instant::now(); let report = probe(&server, "--help", Duration::from_secs(1)).await; - assert!(report.contains("probe timed out after 1s"), "{report}"); + assert!( + report.contains("probe timed out after 1s\nstdout:"), + "{report}" + ); assert!(start.elapsed() < Duration::from_secs(5)); let pid = report .split("stdout:\n") diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index e0f82d8f6a..f0aff72532 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -65,6 +65,31 @@ pub fn reported_engine(serve_stdout: &str) -> Option<&str> { .filter(|engine| !engine.is_empty()) } +/// The engine named on a `rocm serve` command line. +/// +/// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: +/// the CLI then picks a default, which only the printed plan reveals (see +/// [`reported_engine`]). +#[must_use] +pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { + let mut args = args.iter().copied(); + while let Some(arg) = args.next() { + let value = if let Some(inline) = arg.strip_prefix("--engine=") { + Some(inline) + } else if arg == "--engine" { + args.next() + } else { + continue; + }; + return value.filter(|engine| !engine.is_empty()); + } + None +} + +/// What a failed serve's report says when no engine could be determined, so a +/// skipped backend inventory is visible rather than silently absent. +pub const ENGINE_UNKNOWN_EVIDENCE: &str = "engine unknown: backend evidence skipped"; + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -273,8 +298,9 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - reported_engine, serve_attempt_report, service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, + parse_log_path, read_clamped, reported_engine, serve_attempt_report, service_log_tail, + slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -296,6 +322,27 @@ managed service launched } } + #[test] + fn engine_is_read_from_both_flag_forms_and_absent_otherwise() { + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine", "lemonade", "--managed"]), + Some("lemonade") + ); + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine=lemonade", "--managed"]), + Some("lemonade") + ); + for args in [ + &["serve", "m", "--managed"][..], + &["serve", "m", "--engine"], + &["serve", "m", "--engine="], + &["serve", "m", "--engine", ""], + &["serve", "--engine-version=1"], + ] { + assert_eq!(engine_from_serve_args(args), None, "{args:?}"); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 91e0d10d68..86c64fd60e 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,7 +13,8 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, reported_engine, serve_attempt_report, service_log_tail, + ENGINE_UNKNOWN_EVIDENCE, ServeAttempt, archive_service_log, engine_from_serve_args, + reported_engine, serve_attempt_report, service_log_tail, }; /// Model id of the live managed service the reuse scenarios plant. @@ -140,7 +141,7 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// reports the same evidence wherever it happens. async fn serve_failure_evidence( world: &mut E2eWorld, - engine: &str, + engine: Option<&str>, headline: &str, device_state: &str, stdout: &str, @@ -155,8 +156,8 @@ async fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); - let backend_evidence = if engine == "lemonade" { - Some( + let backend_evidence = match engine { + Some("lemonade") => Some( e2e_cucumber::lemonade_diagnostics::backend_evidence( &world .app_paths() @@ -164,9 +165,9 @@ async fn serve_failure_evidence( .join("runtime.json"), ) .await, - ) - } else { - None + ), + Some(_) => None, + None => Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()), }; let stop_status = stop_scenario_services(world); serve_attempt_report(&ServeAttempt { @@ -218,9 +219,7 @@ async fn serve_and_wait( "{}", serve_failure_evidence( world, - args.windows(2) - .find(|pair| pair[0] == "--engine") - .map_or("", |pair| pair[1]), + engine_from_serve_args(args).or_else(|| reported_engine(&stdout)), &headline, &device_state, &stdout, @@ -636,7 +635,7 @@ async fn serve_gpu_model(world: &mut E2eWorld) { diagnostics.push( serve_failure_evidence( world, - engine, + Some(engine), &format!("attempt {attempt} of {attempts} (rc={rc})"), &device_state, &stdout, @@ -1012,7 +1011,7 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { "{}", serve_failure_evidence( world, - reported_engine(&stdout).unwrap_or(""), + reported_engine(&stdout), &headline, &device_state, &stdout,