diff --git a/Cargo.lock b/Cargo.lock index b9c13d8376..8e63220c4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "sha2", "tempfile", "tokio", "toml", diff --git a/docs/testing.md b/docs/testing.md index 525cb9fb99..6ecd44e497 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -815,6 +815,44 @@ Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"); check that line before relying on this coverage. +### Lemonade managed-serve failure evidence + +On a failed serve, the shared serve steps preserve the full service log under +`service-logs/` in the E2E results artifact before stopping the service. The engine +is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if +neither names one, the report says `engine unknown: backend evidence skipped`. +Lemonade failures also include the installed runtime manifest's version/backend, configured +`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints +and `version.txt`, +and each candidate's `--version` and `--help` output. At most eight candidates +are hashed and probed, with two read-only probes per candidate. Each probe retains +up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are +marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw +output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). +Each probe has a five-second timeout. Hashing and probes share a 20-second budget +after inventory discovery; hashing runs off the async executor, rejects files +over 32 MiB, and checks its deadline between 64 KiB reads. Filesystem operations +and process cleanup are not preemptible, so this is not a hard wall-clock limit +on the entire report. +A candidate or configured build pin is **not proof** of which binary the router +launched; a failed probe is retained as evidence, not treated as an identity. +Probes inherit the harness environment rather than reconstructing the engine's +managed library paths, so a probe loader error alone does not establish why the +serve failed. + +Use the evidence to distinguish backend startup, download, and router readiness +failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the +Strix Halo Ubuntu investigation. A backend's ready line does not establish that +the router serves the requested model, and a readiness timeout alone does not +establish a shutdown. Diagnostics do not change the CLI's asynchronous +managed-launch contract or imply a backend fix. + +Exercise the diagnostics and expectation scoping without a GPU: + +```bash +cargo test -p e2e-cucumber --lib lemonade +``` + ## Runtime Selection And Activation List registered managed or read-only runtimes and their exact side-by-side keys: diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 719f32ac35..1c9f9f3b72 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -61,8 +61,9 @@ rand.workspace = true reqwest = { version = "0.13", features = ["json"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tempfile = "3" -tokio.workspace = true +tokio = { workspace = true, features = ["process", "io-util"] } toml = "1.1" # `ServeDir` backs the install scenario's loopback download server (see # `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing, diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index fec5e0154f..60eefd2520 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -43,60 +43,44 @@ when = {} bug = "EAI-7219" reason = "Engines do not report a consistent resolved model name for a short name." -# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: -# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run -# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails -# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry -# `flaky = true` — either outcome is tolerated while the bug stays open. The flag -# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added -# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade -# build a given run used cannot currently be read back from this lane, so an -# XPASS streak here is not by itself evidence that the bug is fixed. +# ROCm/rocm-cli#897 tracks multiple Strix Halo Ubuntu Lemonade serve signatures: +# the child rejects --no-mmap (41984921053), backend readiness on :8001 without +# the model appearing on :11435 (41984503055), and backend download failure +# (41986772728). A backend ready line does not prove a later shutdown. +# EAI-7423 described only the earlier ready-then-unavailable symptom. +# Retain flaky=true because historical runs also passed intermittently; do not +# infer a fix from a single XPASS. Failed serves now capture candidate executable +# --version/--help and configured pins separately, before stopping the service. # -# On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve -# loads the model and reaches ready on :8001, then is shut down ~0.08s later, so -# the endpoint never becomes reachable (EAI-7423, root-caused on the box -# 2026-07-15 — NOT the earlier "backend install hides the plan line" theory, which -# was disproven: the `engine:` line IS emitted). os=linux: on native Windows this -# scenario PASSES (run 29338475133 Strix-Windows), so it must NOT be xfail'd there. +# os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade-native LINUX host the served model's endpoint drops before -# inference: the lemonade managed serve reaches ready then is shut down ~0.08s -# later (EAI-7423, root-caused on the box 2026-07-15 — not a Vulkan hang). -# os=linux: passes on native Windows (run 29338475133). +# The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade LINUX host this scenario does a real lemonade serve, which reaches -# ready then shuts down immediately (EAI-7423, see serve-lemonade-inference). -# os=linux: on native Windows lemonade serve works, so this PASSES there (verified -# run 29338475133 Strix-Windows XPASS) and must not be xfail'd. +# The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is -# shut down immediately, so inference never happens (root-caused on the box -# 2026-07-15; previously mis-attributed to the EAI-7052 Vulkan hang). os=linux: -# native Windows lemonade serve works (run 29338475133 Strix-Windows XPASS'd -# this), so scope the xfail to Linux. --- +# Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -107,16 +91,11 @@ bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# On a lemonade LINUX host this needs a live lemonade serve, which reaches ready -# then shuts down immediately (EAI-7423). os=linux: passes on native Windows -# lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the -# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a -# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -125,15 +104,12 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade -# serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to -# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not -# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# path works. Only the real Lemonade Linux serve stays xfail under #897; +# the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -144,15 +120,7 @@ when = {} bug = "EAI-7384" reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-provisioning." -# The GPU-backed bench scenario benchmarks a really served model, so on a -# lemonade LINUX host it inherits the same managed-serve bug as the sibling -# serve/chat rows below `serve-default-engine-working-endpoint`: the serve -# reaches ready then shuts down immediately, so this scenario's `a model is -# being served on GPU` precondition never yields a live endpoint. Scoped -# identically — os=linux only, because native Windows lemonade serve works, and -# therock_family=gfx* so it applies only on a real GPU host. `flaky = true` -# matches those rows too: which of them pass varies run to run on that lane, and -# a deterministic-XPASS row would fail the lane on unrelated PRs. +# Bench's real-serve precondition is blocked by #897 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -160,7 +128,19 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." +serve_timeout_secs = 90 +flaky = true + +# serve-40 has the same serve precondition; this is not an automation-helper bug. +# xfail + flaky tolerates either outcome, so on this lane the "no background +# helper" assertion carries no signal until #897 is closed: a pass here does not +# show the scenario's own behaviour is healthy, and a failure is not attributed +# to it. Remove the row once #897 is fixed. +[["serve-stop-leaves-no-background-helper"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 539644f521..1dcf18217f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -75,7 +75,7 @@ Feature: Model serving # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found # under bin/llamacpp// — unlike the short-recipe-name path above, whose # managed-lemonade fallback would mask the identical failure behind the - # unrelated EAI-7423 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as + # #897 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index df78c81db4..60c58f495f 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -826,6 +826,39 @@ mod tests { } } + #[test] + fn lemonade_serve_failures_track_897_without_xfailing_mock_or_windows() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + ] { + let scenario = decl(&[&format!("id:{id}")]); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "#897" + ), + "{id}" + ); + assert_eq!( + matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + Some(90), + "{id}", + ); + for platform in ["mock", "strix-windows"] { + assert_eq!( + resolve(&scenario, &cap(platform), &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {platform}", + ); + } + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs new file mode 100644 index 0000000000..0dfcddd7e7 --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -0,0 +1,637 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read-only backend evidence for a failed Lemonade serve, before teardown. + +use sha2::{Digest as _, Sha256}; +use std::fmt::Write as _; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; + +use crate::serve_log::read_clamped; + +const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CANDIDATES: usize = 8; +const TOTAL_EVIDENCE_BUDGET: Duration = Duration::from_secs(20); +const MAX_HASH_BYTES: u64 = 32 * 1024 * 1024; + +/// Record the installed version and pins separately from executable build output: +/// a pin describes what should be installed, not what a runner actually cached. +/// +/// Every candidate is labelled as such; finding a binary does not prove that the +/// router launched it. No install, download, model load, or cache repair is done. +pub async fn backend_evidence(manifest_path: &Path) -> String { + backend_evidence_with_budget(manifest_path, TOTAL_EVIDENCE_BUDGET).await +} + +async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> String { + let mut report = String::new(); + let _ = writeln!(report, "manifest: {}", manifest_path.display()); + let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return format!("{report}unavailable: {error}"), + }; + for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { + let _ = writeln!( + report, + "{key}: {}", + manifest.get(key).unwrap_or(&serde_json::Value::Null) + ); + } + let Some(runtime) = manifest + .get("runtime_dir") + .and_then(serde_json::Value::as_str) + else { + return format!("{report}backend inventory unavailable: no runtime_dir"); + }; + let runtime = Path::new(runtime); + let pins = runtime.join("resources/backend_versions.json"); + let _ = writeln!(report, "configured pins (not executable identity):"); + match read_clamped(&pins, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); + } + Err(error) => { + let _ = writeln!(report, "{}: {error}", pins.display()); + } + } + let candidates = backend_candidates(runtime); + let _ = writeln!( + report, + "probes inherit the harness environment, not the engine's managed library environment; \ + a probe loader failure does not establish a serving failure" + ); + if candidates.is_empty() { + let _ = writeln!( + report, + "no llama-server candidates under bin/llamacpp (backend download may not have completed)" + ); + } + let deadline = Instant::now() + budget; + for server in candidates.iter().take(MAX_CANDIDATES) { + if Instant::now() >= deadline { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let _ = writeln!( + report, + "candidate (not proof of selection): {}", + server.display() + ); + let path = server.clone(); + match tokio::task::spawn_blocking(move || fingerprint(&path, deadline)) + .await + .map_err(std::io::Error::other) + .and_then(|result| result) + { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + } + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let output = probe(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + if candidates.len() > MAX_CANDIDATES { + let _ = writeln!( + report, + "backend inventory truncated after {MAX_CANDIDATES} candidates" + ); + } + report +} + +fn backend_candidates(runtime: &Path) -> Vec { + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut candidates = Vec::new(); + for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { + let dir = runtime.join("bin/llamacpp").join(backend); + let direct = dir.join(binary); + if direct.is_file() { + candidates.push(direct); + } + // Lemonade archives also unpack one level below the backend directory. + if let Ok(entries) = std::fs::read_dir(dir) { + let mut nested: Vec<_> = entries + .filter_map(Result::ok) + .map(|entry| entry.path().join(binary)) + .filter(|path| path.is_file()) + .collect(); + nested.sort(); + candidates.extend(nested); + } + } + candidates +} + +fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + let mut file = std::fs::File::open(server)?; + // A real server is much smaller; do not hash an arbitrarily large corrupt + // cache entry while reporting a failure. Never label a partial hash as SHA256. + if file.metadata()?.len() > MAX_HASH_BYTES { + return Err(std::io::Error::other( + "executable exceeds 32 MiB hash limit", + )); + } + let mut hasher = Sha256::new(); + let mut count = 0; + let mut buffer = vec![0; 64 * 1024]; + loop { + // Check inside the worker, not just around its JoinHandle: timing out a + // spawn_blocking await alone would leave the hashing thread running. + if Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "hash evidence budget exhausted", + )); + } + let read = file.read(&mut buffer)?; + count += read as u64; + if count > MAX_HASH_BYTES { + return Err(std::io::Error::other( + "executable exceeds 32 MiB hash limit", + )); + } + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +#[derive(Default)] +struct CapturedOutput { + bytes: Vec, + truncated: bool, +} + +impl CapturedOutput { + fn truncation_note(&self, stream: &str) -> String { + if self.truncated { + format!("\n<{stream} truncated at 32 KiB>") + } else { + String::new() + } + } +} + +async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { + let mut command = tokio::process::Command::new(server); + command + .arg(flag) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return format!("probe could not start: {error}"), + }; + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let mut out = CapturedOutput::default(); + let mut err = CapturedOutput::default(); + let status = if let Ok((status, stdout_result, stderr_result)) = + tokio::time::timeout(timeout, async { + tokio::join!( + child.wait(), + drain_output(stdout, &mut out), + drain_output(stderr, &mut err), + ) + }) + .await + { + let mut status = status.map_or_else(|error| error.to_string(), |status| status.to_string()); + for (stream, result) in [("stdout", stdout_result), ("stderr", stderr_result)] { + if let Err(error) = result { + let _ = write!(status, "\n{stream} capture failed: {error}"); + } + } + status + } else { + // kill_on_drop also covers cancellation of the whole reporter. + // Explicitly wait here so a normal timeout leaves no zombie child. + let mut status = format!("probe timed out after {}s", timeout.as_secs_f64()); + if let Err(error) = child.kill().await { + let _ = write!(status, "; kill/reap failed: {error}"); + } + status + }; + format!( + "{status}\nstdout:\n{}{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out.bytes), + out.truncation_note("stdout"), + String::from_utf8_lossy(&err.bytes), + err.truncation_note("stderr"), + ) +} + +async fn drain_output( + mut reader: impl AsyncRead + Unpin, + retained: &mut CapturedOutput, +) -> std::io::Result<()> { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + let mut buffer = [0; 4096]; + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + // Continue draining both pipes after the cap, without retaining more + // bytes or writing them to disk. A noisy child must not block on output. + let keep = count.min(limit.saturating_sub(retained.bytes.len())); + retained.bytes.extend_from_slice(&buffer[..keep]); + retained.truncated |= keep < count; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_manifest(runtime: &Path) -> PathBuf { + let manifest = runtime.join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + manifest + } + + #[test] + fn executable_identity_does_not_depend_on_the_dynamic_loader() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("not-executable"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + let file = std::fs::File::create(&server).unwrap(); + file.set_len(MAX_HASH_BYTES).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap() + .len(), + 64 + ); + file.set_len(MAX_HASH_BYTES + 1).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap_err() + .to_string(), + "executable exceeds 32 MiB hash limit" + ); + } + + #[test] + fn hash_worker_obeys_its_own_deadline() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now()).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + + #[test] + fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut wanted = Vec::new(); + for relative in ["rocm-stable", "rocm-nightly/llama-b9752", "vulkan/build"] { + let server = root.path().join("bin/llamacpp").join(relative).join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, "").unwrap(); + wanted.push(server); + } + let cpu = root.path().join("bin/llamacpp/cpu").join(binary); + std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); + std::fs::write(cpu, "").unwrap(); + assert_eq!(backend_candidates(root.path()), wanted); + } + + #[tokio::test] + async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + for i in 0..=MAX_CANDIDATES { + let server = root + .path() + .join(format!("bin/llamacpp/rocm-stable/{i:02}/{binary}")); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + let report = backend_evidence(&write_manifest(root.path())).await; + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + assert_eq!(report.matches("sha256:").count(), MAX_CANDIDATES); + assert_eq!(report.matches("--help:").count(), MAX_CANDIDATES); + assert!(report.contains("backend inventory truncated after 8 candidates")); + let skipped = root + .path() + .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); + assert!(!report.contains(&skipped.display().to_string()), "{report}"); + } + + #[tokio::test] + async fn exhausted_budget_skips_hashing_and_probes() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::ZERO).await; + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + for skipped in ["sha256", "--version:", "--help:", "candidate ("] { + assert!(!report.contains(skipped), "{report}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("bin/llamacpp/rocm-stable/llama-server"); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nexec /bin/sleep 30\n", + ) + .unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::from_secs(1)) + .await; + assert!(report.contains("--version:\nprobe timed out"), "{report}"); + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + assert!(!report.contains("--help:"), "{report}"); + } + + #[tokio::test] + async fn stream_capture_marks_only_bytes_actually_discarded() { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + for length in [0, limit - 1, limit, limit + 1] { + let bytes = vec![b'x'; length]; + let mut captured = CapturedOutput::default(); + drain_output(bytes.as_slice(), &mut captured).await.unwrap(); + assert_eq!(captured.bytes, vec![b'x'; length.min(limit)]); + assert_eq!(captured.truncated, length > limit); + assert_eq!( + captured.truncation_note("stdout"), + if length > limit { + "\n" + } else { + "" + } + ); + } + } + + #[tokio::test] + async fn stream_truncation_survives_timeout_before_eof() { + use tokio::io::AsyncWriteExt as _; + let length = MAX_EVIDENCE_BYTES as usize / 2 + 1; + let (reader, mut writer) = tokio::io::duplex(length); + writer.write_all(&vec![b'x'; length]).await.unwrap(); + let mut captured = CapturedOutput::default(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + drain_output(reader, &mut captured) + ) + .await + .is_err() + ); + assert_eq!(captured.bytes.len(), length - 1); + assert_eq!( + captured.truncation_note("stderr"), + "\n" + ); + } + + #[tokio::test] + async fn missing_manifest_is_reported_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let report = backend_evidence(&root.path().join("missing.json")).await; + assert!(report.contains("unavailable:"), "{report}"); + } + + #[tokio::test] + async fn failed_download_keeps_pins_but_does_not_invent_an_installed_build() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + std::fs::create_dir_all(runtime.join("resources")).unwrap(); + std::fs::write( + runtime.join("resources/backend_versions.json"), + r#"{"llamacpp":{"rocm-stable":"b9752"}}"#, + ) + .unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!( + report.contains("configured pins (not executable identity):"), + "{report}" + ); + assert!(report.contains("\"rocm-stable\":\"b9752\""), "{report}"); + assert!(report.contains("no llama-server candidates"), "{report}"); + } + + #[tokio::test] + async fn probe_start_error_is_evidence_not_a_second_failure() { + let root = tempfile::tempdir().unwrap(); + let report = probe(&root.path().join("missing"), "--version", PROBE_TIMEOUT).await; + assert!(report.contains("probe could not start:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn inventory_probes_the_cached_executable_not_a_path_lookup() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + let dir = runtime.join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(&dir).unwrap(); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "\ +#!/bin/sh +case \"$1\" in + --version) echo 'version: 9752 (fixture-build)' ;; + --help) echo '--mmap, --no-mmap'; echo 'fixture help stderr' >&2; exit 2 ;; + *) exit 99 ;; +esac +", + ) + .unwrap(); + std::fs::write(dir.join("version.txt"), "cached-build").unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); + assert!(report.contains("version.txt: cached-build"), "{report}"); + assert!( + report.contains("--help:\nexit status: 2\nstdout:"), + "{report}" + ); + assert!(!report.contains("capture"), "{report}"); + assert!(!report.contains("Ok("), "{report}"); + assert!(report.contains("--mmap, --no-mmap"), "{report}"); + assert!(report.contains("fixture help stderr"), "{report}"); + assert!( + server.is_file(), + "diagnostics must not repair or remove the cache" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn probe_timeout_is_bounded() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + // exec keeps the sleeper in the child's PID, so cancellation kills it + // rather than leaving a grandchild behind. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\necho $$\nexec /bin/sleep 30\n", + ) + .unwrap(); + let start = std::time::Instant::now(); + let report = probe(&server, "--help", Duration::from_secs(1)).await; + assert!( + report.contains("probe timed out after 1s\nstdout:"), + "{report}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + let pid = report + .split("stdout:\n") + .nth(1) + .unwrap() + .lines() + .next() + .unwrap() + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn exact_stream_limit_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 32768 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains("exit status: 0"), "{report}"); + assert_eq!(report.bytes().filter(|byte| *byte == 0).count(), 32768); + assert!(!report.contains("capped"), "{report}"); + assert!(!report.contains("truncated"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn noisy_probe_output_is_capped() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 100000 /dev/zero\nhead -c 100000 /dev/zero >&2\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains(""), "{report}"); + assert!(report.ends_with(""), "{report}"); + assert_eq!( + report.bytes().filter(|byte| *byte == 0).count(), + MAX_EVIDENCE_BYTES as usize + ); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 300); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 6d16d6c4dd..f56a347bfa 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -9,6 +9,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_diagnostics; pub mod loopback_http; pub mod mock_server; pub mod model_id; diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index bf0ff9d8c1..f0aff72532 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -55,6 +55,41 @@ fn parse_log_path(serve_stdout: &str) -> Option<&str> { .filter(|path| !path.is_empty()) } +/// The selected engine, if the serve reached the point of printing its plan. +#[must_use] +pub fn reported_engine(serve_stdout: &str) -> Option<&str> { + serve_stdout + .lines() + .find_map(|line| line.trim().strip_prefix("engine:")) + .map(str::trim) + .filter(|engine| !engine.is_empty()) +} + +/// The engine named on a `rocm serve` command line. +/// +/// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: +/// the CLI then picks a default, which only the printed plan reveals (see +/// [`reported_engine`]). +#[must_use] +pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { + let mut args = args.iter().copied(); + while let Some(arg) = args.next() { + let value = if let Some(inline) = arg.strip_prefix("--engine=") { + Some(inline) + } else if arg == "--engine" { + args.next() + } else { + continue; + }; + return value.filter(|engine| !engine.is_empty()); + } + None +} + +/// What a failed serve's report says when no engine could be determined, so a +/// skipped backend inventory is visible rather than silently absent. +pub const ENGINE_UNKNOWN_EVIDENCE: &str = "engine unknown: backend evidence skipped"; + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -120,14 +155,14 @@ fn slugify(name: &str) -> String { /// /// The log may still be being written, so `len` is a snapshot: the head and tail /// are always real, and only the elided count can be slightly stale. -fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { +pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; let mut file = std::fs::File::open(path)?; let len = file.metadata()?.len(); if len <= max { let mut body = Vec::new(); - file.read_to_end(&mut body)?; + file.take(max).read_to_end(&mut body)?; return Ok((body, String::new())); } // `len > max >= 2 * half`, so the two halves can never overlap. @@ -230,6 +265,8 @@ pub struct ServeAttempt<'a> { /// failed, means the engine may still hold the port and the device — which /// changes how every later failure in the run should be read. pub stop_status: &'a str, + /// Only Lemonade attempts carry the optional backend inventory. + pub backend_evidence: Option<&'a str>, } /// Render one serve attempt's evidence as a failure message. @@ -241,7 +278,7 @@ pub struct ServeAttempt<'a> { #[must_use] pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { use crate::section; - format!( + let mut report = format!( "{}\n{}\n{}\n{}\n{}\n{}\n{}", attempt.headline, section("device state", attempt.device_state), @@ -250,14 +287,20 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { section("service log (tail)", attempt.log_tail), section("service log (full)", attempt.archived_log), section("stop", attempt.stop_status), - ) + ); + if let Some(evidence) = attempt.backend_evidence { + report.push_str("\n--- Lemonade backend evidence ---\n"); + report.push_str(evidence); + } + report } #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - serve_attempt_report, service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, + parse_log_path, read_clamped, reported_engine, serve_attempt_report, service_log_tail, + slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -270,6 +313,36 @@ managed service launched readiness: starting "; + #[test] + fn engine_lookup_is_safe_for_missing_or_incomplete_serve_plans() { + assert_eq!(reported_engine(SERVE_PLAN), Some("vllm")); + assert_eq!(reported_engine(" engine: lemonade\n"), Some("lemonade")); + for output in ["", "serve failed", " engine: \n"] { + assert_eq!(reported_engine(output), None); + } + } + + #[test] + fn engine_is_read_from_both_flag_forms_and_absent_otherwise() { + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine", "lemonade", "--managed"]), + Some("lemonade") + ); + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine=lemonade", "--managed"]), + Some("lemonade") + ); + for args in [ + &["serve", "m", "--managed"][..], + &["serve", "m", "--engine"], + &["serve", "m", "--engine="], + &["serve", "m", "--engine", ""], + &["serve", "--engine-version=1"], + ] { + assert_eq!(engine_from_serve_args(args), None, "{args:?}"); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( @@ -502,7 +575,7 @@ managed service launched /// full log went — none of which the plain readiness timeout reported. #[test] fn a_stalled_serve_report_carries_every_piece_of_evidence() { - let report = serve_attempt_report(&ServeAttempt { + let mut attempt = ServeAttempt { headline: "`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed` \ exited 0, but the endpoint never served `Qwen3-0.6B` within 600s", device_state: "device state: drained (48000 MiB free of 49000 MiB, floor 44100 MiB)", @@ -511,7 +584,15 @@ managed service launched log_tail: "load: failed to load model", archived_log: "archived to service-logs/scenario--svc.log", stop_status: "lemonade-qwen: stopped", - }); + backend_evidence: None, + }; + let report = serve_attempt_report(&attempt); + assert!(!report.contains("Lemonade backend evidence"), "{report}"); + attempt.backend_evidence = Some("fixture backend identity"); + assert_eq!( + serve_attempt_report(&attempt), + format!("{report}\n--- Lemonade backend evidence ---\nfixture backend identity") + ); for expected in [ "exited 0", diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 521629c8ed..86c64fd60e 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,7 +13,8 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, + ENGINE_UNKNOWN_EVIDENCE, ServeAttempt, archive_service_log, engine_from_serve_args, + reported_engine, serve_attempt_report, service_log_tail, }; /// Model id of the live managed service the reuse scenarios plant. @@ -138,8 +139,9 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( - world: &E2eWorld, +async fn serve_failure_evidence( + world: &mut E2eWorld, + engine: Option<&str>, headline: &str, device_state: &str, stdout: &str, @@ -154,6 +156,19 @@ fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); + let backend_evidence = match engine { + Some("lemonade") => Some( + e2e_cucumber::lemonade_diagnostics::backend_evidence( + &world + .app_paths() + .engine_manifests_dir("lemonade") + .join("runtime.json"), + ) + .await, + ), + Some(_) => None, + None => Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()), + }; let stop_status = stop_scenario_services(world); serve_attempt_report(&ServeAttempt { headline, @@ -163,6 +178,7 @@ fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, + backend_evidence: backend_evidence.as_deref(), }) } @@ -201,7 +217,15 @@ async fn serve_and_wait( }; panic!( "{}", - serve_failure_evidence(world, &headline, &device_state, &stdout, &stderr) + serve_failure_evidence( + world, + engine_from_serve_args(args).or_else(|| reported_engine(&stdout)), + &headline, + &device_state, + &stdout, + &stderr, + ) + .await ); } @@ -608,13 +632,17 @@ async fn serve_gpu_model(world: &mut E2eWorld) { // outcome is quoted: a stop that found no record (the launch had not yet // written one) or failed means the next attempt did NOT get a clean // device, and the failure says so instead of looking like a broken serve. - diagnostics.push(serve_failure_evidence( - world, - &format!("attempt {attempt} of {attempts} (rc={rc})"), - &device_state, - &stdout, - &stderr, - )); + diagnostics.push( + serve_failure_evidence( + world, + Some(engine), + &format!("attempt {attempt} of {attempts} (rc={rc})"), + &device_state, + &stdout, + &stderr, + ) + .await, + ); if attempt == attempts { break; } @@ -977,15 +1005,19 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { &ready_substr, timeout_secs, ); + let stdout = world.cli_output.clone().unwrap_or_default(); + let stderr = world.cli_stderr.clone().unwrap_or_default(); panic!( "{}", serve_failure_evidence( world, + reported_engine(&stdout), &headline, &device_state, - world.cli_output.as_deref().unwrap_or_default(), - world.cli_stderr.as_deref().unwrap_or_default(), + &stdout, + &stderr, ) + .await ); } } @@ -2438,11 +2470,7 @@ async fn assert_endpoint_port(world: &mut E2eWorld) { /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { - let Some(engine) = output - .lines() - .find_map(|l| l.trim().strip_prefix("engine:")) - .map(str::trim) - else { + let Some(engine) = reported_engine(output) else { panic!("no 'engine:' line in serve output:\n{output}"); }; engine