From a5a724fe3baa6fbfc9593f843317ee47d5a5f4e5 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 11:15:58 +0000 Subject: [PATCH 01/28] test(e2e): capture Lemonade backend identity on serve failures Strix Halo Ubuntu managed serves can return success without exposing the model, with logs showing argument rejection, backend-ready/router-unavailable, or download failure. The existing xfails attribute all three to an immediate shutdown. Capture the runtime version, configured pins, candidate backend paths, SHA-256 fingerprints and version/help output alongside the archived service logs before cleanup. Probe duration and retained output are bounded, hashing is capped and runs off the async executor, and timed-out children are killed and reaped. The engine is read from both `--engine X` and `--engine=X`, falls back to the printed serve plan, and the failure report says so when no engine can be determined. Probe capture and kill/reap results are printed only when they fail. Replace the shutdown-only xfail attribution with the observed failure signatures and cover serve-40's shared serve precondition. Intermittent-failure handling and platform scope are unchanged. This is diagnostic coverage, not a confirmed backend fix: configured pins and candidate binaries are evidence of what could have run, not proof of what did. Refs #897 Signed-off-by: Jussi Elo --- Cargo.lock | 1 + docs/testing.md | 38 ++ tests/e2e-cucumber/Cargo.toml | 3 +- tests/e2e-cucumber/expectations.toml | 104 ++- .../features/model_serving.feature | 2 +- tests/e2e-cucumber/src/expectation.rs | 33 + .../e2e-cucumber/src/lemonade_diagnostics.rs | 637 ++++++++++++++++++ tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/serve_log.rs | 97 ++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 64 +- 10 files changed, 890 insertions(+), 90 deletions(-) create mode 100644 tests/e2e-cucumber/src/lemonade_diagnostics.rs diff --git a/Cargo.lock b/Cargo.lock index b9c13d8376..8e63220c4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "sha2", "tempfile", "tokio", "toml", diff --git a/docs/testing.md b/docs/testing.md index 525cb9fb99..6ecd44e497 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -815,6 +815,44 @@ Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"); check that line before relying on this coverage. +### Lemonade managed-serve failure evidence + +On a failed serve, the shared serve steps preserve the full service log under +`service-logs/` in the E2E results artifact before stopping the service. The engine +is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if +neither names one, the report says `engine unknown: backend evidence skipped`. +Lemonade failures also include the installed runtime manifest's version/backend, configured +`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints +and `version.txt`, +and each candidate's `--version` and `--help` output. At most eight candidates +are hashed and probed, with two read-only probes per candidate. Each probe retains +up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are +marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw +output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). +Each probe has a five-second timeout. Hashing and probes share a 20-second budget +after inventory discovery; hashing runs off the async executor, rejects files +over 32 MiB, and checks its deadline between 64 KiB reads. Filesystem operations +and process cleanup are not preemptible, so this is not a hard wall-clock limit +on the entire report. +A candidate or configured build pin is **not proof** of which binary the router +launched; a failed probe is retained as evidence, not treated as an identity. +Probes inherit the harness environment rather than reconstructing the engine's +managed library paths, so a probe loader error alone does not establish why the +serve failed. + +Use the evidence to distinguish backend startup, download, and router readiness +failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the +Strix Halo Ubuntu investigation. A backend's ready line does not establish that +the router serves the requested model, and a readiness timeout alone does not +establish a shutdown. Diagnostics do not change the CLI's asynchronous +managed-launch contract or imply a backend fix. + +Exercise the diagnostics and expectation scoping without a GPU: + +```bash +cargo test -p e2e-cucumber --lib lemonade +``` + ## Runtime Selection And Activation List registered managed or read-only runtimes and their exact side-by-side keys: diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 719f32ac35..1c9f9f3b72 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -61,8 +61,9 @@ rand.workspace = true reqwest = { version = "0.13", features = ["json"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tempfile = "3" -tokio.workspace = true +tokio = { workspace = true, features = ["process", "io-util"] } toml = "1.1" # `ServeDir` backs the install scenario's loopback download server (see # `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing, diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 2fe2e249d9..3acdaf3533 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -58,60 +58,44 @@ when = {} bug = "EAI-7219" reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." -# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: -# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run -# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails -# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry -# `flaky = true` — either outcome is tolerated while the bug stays open. The flag -# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added -# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade -# build a given run used cannot currently be read back from this lane, so an -# XPASS streak here is not by itself evidence that the bug is fixed. +# ROCm/rocm-cli#897 tracks multiple Strix Halo Ubuntu Lemonade serve signatures: +# the child rejects --no-mmap (41984921053), backend readiness on :8001 without +# the model appearing on :11435 (41984503055), and backend download failure +# (41986772728). A backend ready line does not prove a later shutdown. +# EAI-7423 described only the earlier ready-then-unavailable symptom. +# Retain flaky=true because historical runs also passed intermittently; do not +# infer a fix from a single XPASS. Failed serves now capture candidate executable +# --version/--help and configured pins separately, before stopping the service. # -# On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve -# loads the model and reaches ready on :8001, then is shut down ~0.08s later, so -# the endpoint never becomes reachable (EAI-7423, root-caused on the box -# 2026-07-15 — NOT the earlier "backend install hides the plan line" theory, which -# was disproven: the `engine:` line IS emitted). os=linux: on native Windows this -# scenario PASSES (run 29338475133 Strix-Windows), so it must NOT be xfail'd there. +# os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade-native LINUX host the served model's endpoint drops before -# inference: the lemonade managed serve reaches ready then is shut down ~0.08s -# later (EAI-7423, root-caused on the box 2026-07-15 — not a Vulkan hang). -# os=linux: passes on native Windows (run 29338475133). +# The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade LINUX host this scenario does a real lemonade serve, which reaches -# ready then shuts down immediately (EAI-7423, see serve-lemonade-inference). -# os=linux: on native Windows lemonade serve works, so this PASSES there (verified -# run 29338475133 Strix-Windows XPASS) and must not be xfail'd. +# The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is -# shut down immediately, so inference never happens (root-caused on the box -# 2026-07-15; previously mis-attributed to the EAI-7052 Vulkan hang). os=linux: -# native Windows lemonade serve works (run 29338475133 Strix-Windows XPASS'd -# this), so scope the xfail to Linux. --- +# Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -122,16 +106,11 @@ bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# On a lemonade LINUX host this needs a live lemonade serve, which reaches ready -# then shuts down immediately (EAI-7423). os=linux: passes on native Windows -# lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the -# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a -# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -140,15 +119,12 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade -# serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to -# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not -# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# path works. Only the real Lemonade Linux serve stays xfail under #897; +# the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -159,15 +135,7 @@ when = {} bug = "EAI-7384" reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-provisioning." -# The GPU-backed bench scenario benchmarks a really served model, so on a -# lemonade LINUX host it inherits the same managed-serve bug as the sibling -# serve/chat rows below `serve-default-engine-working-endpoint`: the serve -# reaches ready then shuts down immediately, so this scenario's `a model is -# being served on GPU` precondition never yields a live endpoint. Scoped -# identically — os=linux only, because native Windows lemonade serve works, and -# therock_family=gfx* so it applies only on a real GPU host. `flaky = true` -# matches those rows too: which of them pass varies run to run on that lane, and -# a deterministic-XPASS row would fail the lane on unrelated PRs. +# Bench's real-serve precondition is blocked by #897 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -175,7 +143,19 @@ reason = "Managed runtime path nests recursively (runtimes/wheel/...) on re-prov # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." +serve_timeout_secs = 90 +flaky = true + +# serve-40 has the same serve precondition; this is not an automation-helper bug. +# xfail + flaky tolerates either outcome, so on this lane the "no background +# helper" assertion carries no signal until #897 is closed: a pass here does not +# show the scenario's own behaviour is healthy, and a failure is not attributed +# to it. Remove the row once #897 is fixed. +[["serve-stop-leaves-no-background-helper"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "#897" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 539644f521..1dcf18217f 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -75,7 +75,7 @@ Feature: Model serving # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found # under bin/llamacpp// — unlike the short-recipe-name path above, whose # managed-lemonade fallback would mask the identical failure behind the - # unrelated EAI-7423 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as + # #897 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index df78c81db4..60c58f495f 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -826,6 +826,39 @@ mod tests { } } + #[test] + fn lemonade_serve_failures_track_897_without_xfailing_mock_or_windows() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + ] { + let scenario = decl(&[&format!("id:{id}")]); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "#897" + ), + "{id}" + ); + assert_eq!( + matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + Some(90), + "{id}", + ); + for platform in ["mock", "strix-windows"] { + assert_eq!( + resolve(&scenario, &cap(platform), &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {platform}", + ); + } + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs new file mode 100644 index 0000000000..0dfcddd7e7 --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -0,0 +1,637 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read-only backend evidence for a failed Lemonade serve, before teardown. + +use sha2::{Digest as _, Sha256}; +use std::fmt::Write as _; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; + +use crate::serve_log::read_clamped; + +const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CANDIDATES: usize = 8; +const TOTAL_EVIDENCE_BUDGET: Duration = Duration::from_secs(20); +const MAX_HASH_BYTES: u64 = 32 * 1024 * 1024; + +/// Record the installed version and pins separately from executable build output: +/// a pin describes what should be installed, not what a runner actually cached. +/// +/// Every candidate is labelled as such; finding a binary does not prove that the +/// router launched it. No install, download, model load, or cache repair is done. +pub async fn backend_evidence(manifest_path: &Path) -> String { + backend_evidence_with_budget(manifest_path, TOTAL_EVIDENCE_BUDGET).await +} + +async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> String { + let mut report = String::new(); + let _ = writeln!(report, "manifest: {}", manifest_path.display()); + let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return format!("{report}unavailable: {error}"), + }; + for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { + let _ = writeln!( + report, + "{key}: {}", + manifest.get(key).unwrap_or(&serde_json::Value::Null) + ); + } + let Some(runtime) = manifest + .get("runtime_dir") + .and_then(serde_json::Value::as_str) + else { + return format!("{report}backend inventory unavailable: no runtime_dir"); + }; + let runtime = Path::new(runtime); + let pins = runtime.join("resources/backend_versions.json"); + let _ = writeln!(report, "configured pins (not executable identity):"); + match read_clamped(&pins, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); + } + Err(error) => { + let _ = writeln!(report, "{}: {error}", pins.display()); + } + } + let candidates = backend_candidates(runtime); + let _ = writeln!( + report, + "probes inherit the harness environment, not the engine's managed library environment; \ + a probe loader failure does not establish a serving failure" + ); + if candidates.is_empty() { + let _ = writeln!( + report, + "no llama-server candidates under bin/llamacpp (backend download may not have completed)" + ); + } + let deadline = Instant::now() + budget; + for server in candidates.iter().take(MAX_CANDIDATES) { + if Instant::now() >= deadline { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let _ = writeln!( + report, + "candidate (not proof of selection): {}", + server.display() + ); + let path = server.clone(); + match tokio::task::spawn_blocking(move || fingerprint(&path, deadline)) + .await + .map_err(std::io::Error::other) + .and_then(|result| result) + { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + } + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let output = probe(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + if candidates.len() > MAX_CANDIDATES { + let _ = writeln!( + report, + "backend inventory truncated after {MAX_CANDIDATES} candidates" + ); + } + report +} + +fn backend_candidates(runtime: &Path) -> Vec { + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut candidates = Vec::new(); + for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { + let dir = runtime.join("bin/llamacpp").join(backend); + let direct = dir.join(binary); + if direct.is_file() { + candidates.push(direct); + } + // Lemonade archives also unpack one level below the backend directory. + if let Ok(entries) = std::fs::read_dir(dir) { + let mut nested: Vec<_> = entries + .filter_map(Result::ok) + .map(|entry| entry.path().join(binary)) + .filter(|path| path.is_file()) + .collect(); + nested.sort(); + candidates.extend(nested); + } + } + candidates +} + +fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + let mut file = std::fs::File::open(server)?; + // A real server is much smaller; do not hash an arbitrarily large corrupt + // cache entry while reporting a failure. Never label a partial hash as SHA256. + if file.metadata()?.len() > MAX_HASH_BYTES { + return Err(std::io::Error::other( + "executable exceeds 32 MiB hash limit", + )); + } + let mut hasher = Sha256::new(); + let mut count = 0; + let mut buffer = vec![0; 64 * 1024]; + loop { + // Check inside the worker, not just around its JoinHandle: timing out a + // spawn_blocking await alone would leave the hashing thread running. + if Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "hash evidence budget exhausted", + )); + } + let read = file.read(&mut buffer)?; + count += read as u64; + if count > MAX_HASH_BYTES { + return Err(std::io::Error::other( + "executable exceeds 32 MiB hash limit", + )); + } + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +#[derive(Default)] +struct CapturedOutput { + bytes: Vec, + truncated: bool, +} + +impl CapturedOutput { + fn truncation_note(&self, stream: &str) -> String { + if self.truncated { + format!("\n<{stream} truncated at 32 KiB>") + } else { + String::new() + } + } +} + +async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { + let mut command = tokio::process::Command::new(server); + command + .arg(flag) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return format!("probe could not start: {error}"), + }; + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let mut out = CapturedOutput::default(); + let mut err = CapturedOutput::default(); + let status = if let Ok((status, stdout_result, stderr_result)) = + tokio::time::timeout(timeout, async { + tokio::join!( + child.wait(), + drain_output(stdout, &mut out), + drain_output(stderr, &mut err), + ) + }) + .await + { + let mut status = status.map_or_else(|error| error.to_string(), |status| status.to_string()); + for (stream, result) in [("stdout", stdout_result), ("stderr", stderr_result)] { + if let Err(error) = result { + let _ = write!(status, "\n{stream} capture failed: {error}"); + } + } + status + } else { + // kill_on_drop also covers cancellation of the whole reporter. + // Explicitly wait here so a normal timeout leaves no zombie child. + let mut status = format!("probe timed out after {}s", timeout.as_secs_f64()); + if let Err(error) = child.kill().await { + let _ = write!(status, "; kill/reap failed: {error}"); + } + status + }; + format!( + "{status}\nstdout:\n{}{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out.bytes), + out.truncation_note("stdout"), + String::from_utf8_lossy(&err.bytes), + err.truncation_note("stderr"), + ) +} + +async fn drain_output( + mut reader: impl AsyncRead + Unpin, + retained: &mut CapturedOutput, +) -> std::io::Result<()> { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + let mut buffer = [0; 4096]; + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + // Continue draining both pipes after the cap, without retaining more + // bytes or writing them to disk. A noisy child must not block on output. + let keep = count.min(limit.saturating_sub(retained.bytes.len())); + retained.bytes.extend_from_slice(&buffer[..keep]); + retained.truncated |= keep < count; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn write_manifest(runtime: &Path) -> PathBuf { + let manifest = runtime.join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + manifest + } + + #[test] + fn executable_identity_does_not_depend_on_the_dynamic_loader() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("not-executable"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + let file = std::fs::File::create(&server).unwrap(); + file.set_len(MAX_HASH_BYTES).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap() + .len(), + 64 + ); + file.set_len(MAX_HASH_BYTES + 1).unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + .unwrap_err() + .to_string(), + "executable exceeds 32 MiB hash limit" + ); + } + + #[test] + fn hash_worker_obeys_its_own_deadline() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now()).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + + #[test] + fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut wanted = Vec::new(); + for relative in ["rocm-stable", "rocm-nightly/llama-b9752", "vulkan/build"] { + let server = root.path().join("bin/llamacpp").join(relative).join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, "").unwrap(); + wanted.push(server); + } + let cpu = root.path().join("bin/llamacpp/cpu").join(binary); + std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); + std::fs::write(cpu, "").unwrap(); + assert_eq!(backend_candidates(root.path()), wanted); + } + + #[tokio::test] + async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + for i in 0..=MAX_CANDIDATES { + let server = root + .path() + .join(format!("bin/llamacpp/rocm-stable/{i:02}/{binary}")); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + let report = backend_evidence(&write_manifest(root.path())).await; + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + assert_eq!(report.matches("sha256:").count(), MAX_CANDIDATES); + assert_eq!(report.matches("--help:").count(), MAX_CANDIDATES); + assert!(report.contains("backend inventory truncated after 8 candidates")); + let skipped = root + .path() + .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); + assert!(!report.contains(&skipped.display().to_string()), "{report}"); + } + + #[tokio::test] + async fn exhausted_budget_skips_hashing_and_probes() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::ZERO).await; + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + for skipped in ["sha256", "--version:", "--help:", "candidate ("] { + assert!(!report.contains(skipped), "{report}"); + } + } + + #[cfg(unix)] + #[tokio::test] + async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("bin/llamacpp/rocm-stable/llama-server"); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nexec /bin/sleep 30\n", + ) + .unwrap(); + let report = + backend_evidence_with_budget(&write_manifest(root.path()), Duration::from_secs(1)) + .await; + assert!(report.contains("--version:\nprobe timed out"), "{report}"); + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + assert!(!report.contains("--help:"), "{report}"); + } + + #[tokio::test] + async fn stream_capture_marks_only_bytes_actually_discarded() { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + for length in [0, limit - 1, limit, limit + 1] { + let bytes = vec![b'x'; length]; + let mut captured = CapturedOutput::default(); + drain_output(bytes.as_slice(), &mut captured).await.unwrap(); + assert_eq!(captured.bytes, vec![b'x'; length.min(limit)]); + assert_eq!(captured.truncated, length > limit); + assert_eq!( + captured.truncation_note("stdout"), + if length > limit { + "\n" + } else { + "" + } + ); + } + } + + #[tokio::test] + async fn stream_truncation_survives_timeout_before_eof() { + use tokio::io::AsyncWriteExt as _; + let length = MAX_EVIDENCE_BYTES as usize / 2 + 1; + let (reader, mut writer) = tokio::io::duplex(length); + writer.write_all(&vec![b'x'; length]).await.unwrap(); + let mut captured = CapturedOutput::default(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + drain_output(reader, &mut captured) + ) + .await + .is_err() + ); + assert_eq!(captured.bytes.len(), length - 1); + assert_eq!( + captured.truncation_note("stderr"), + "\n" + ); + } + + #[tokio::test] + async fn missing_manifest_is_reported_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let report = backend_evidence(&root.path().join("missing.json")).await; + assert!(report.contains("unavailable:"), "{report}"); + } + + #[tokio::test] + async fn failed_download_keeps_pins_but_does_not_invent_an_installed_build() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + std::fs::create_dir_all(runtime.join("resources")).unwrap(); + std::fs::write( + runtime.join("resources/backend_versions.json"), + r#"{"llamacpp":{"rocm-stable":"b9752"}}"#, + ) + .unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!( + report.contains("configured pins (not executable identity):"), + "{report}" + ); + assert!(report.contains("\"rocm-stable\":\"b9752\""), "{report}"); + assert!(report.contains("no llama-server candidates"), "{report}"); + } + + #[tokio::test] + async fn probe_start_error_is_evidence_not_a_second_failure() { + let root = tempfile::tempdir().unwrap(); + let report = probe(&root.path().join("missing"), "--version", PROBE_TIMEOUT).await; + assert!(report.contains("probe could not start:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn inventory_probes_the_cached_executable_not_a_path_lookup() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + let dir = runtime.join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(&dir).unwrap(); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "\ +#!/bin/sh +case \"$1\" in + --version) echo 'version: 9752 (fixture-build)' ;; + --help) echo '--mmap, --no-mmap'; echo 'fixture help stderr' >&2; exit 2 ;; + *) exit 99 ;; +esac +", + ) + .unwrap(); + std::fs::write(dir.join("version.txt"), "cached-build").unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = backend_evidence(&manifest).await; + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); + assert!(report.contains("version.txt: cached-build"), "{report}"); + assert!( + report.contains("--help:\nexit status: 2\nstdout:"), + "{report}" + ); + assert!(!report.contains("capture"), "{report}"); + assert!(!report.contains("Ok("), "{report}"); + assert!(report.contains("--mmap, --no-mmap"), "{report}"); + assert!(report.contains("fixture help stderr"), "{report}"); + assert!( + server.is_file(), + "diagnostics must not repair or remove the cache" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn probe_timeout_is_bounded() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + // exec keeps the sleeper in the child's PID, so cancellation kills it + // rather than leaving a grandchild behind. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\necho $$\nexec /bin/sleep 30\n", + ) + .unwrap(); + let start = std::time::Instant::now(); + let report = probe(&server, "--help", Duration::from_secs(1)).await; + assert!( + report.contains("probe timed out after 1s\nstdout:"), + "{report}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + let pid = report + .split("stdout:\n") + .nth(1) + .unwrap() + .lines() + .next() + .unwrap() + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn exact_stream_limit_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 32768 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains("exit status: 0"), "{report}"); + assert_eq!(report.bytes().filter(|byte| *byte == 0).count(), 32768); + assert!(!report.contains("capped"), "{report}"); + assert!(!report.contains("truncated"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn noisy_probe_output_is_capped() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 100000 /dev/zero\nhead -c 100000 /dev/zero >&2\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains(""), "{report}"); + assert!(report.ends_with(""), "{report}"); + assert_eq!( + report.bytes().filter(|byte| *byte == 0).count(), + MAX_EVIDENCE_BYTES as usize + ); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 300); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 562305549d..6f4aaf58f5 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -9,6 +9,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_diagnostics; pub mod loopback_http; pub mod mock_server; pub mod model_id; diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index bf0ff9d8c1..f0aff72532 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -55,6 +55,41 @@ fn parse_log_path(serve_stdout: &str) -> Option<&str> { .filter(|path| !path.is_empty()) } +/// The selected engine, if the serve reached the point of printing its plan. +#[must_use] +pub fn reported_engine(serve_stdout: &str) -> Option<&str> { + serve_stdout + .lines() + .find_map(|line| line.trim().strip_prefix("engine:")) + .map(str::trim) + .filter(|engine| !engine.is_empty()) +} + +/// The engine named on a `rocm serve` command line. +/// +/// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: +/// the CLI then picks a default, which only the printed plan reveals (see +/// [`reported_engine`]). +#[must_use] +pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { + let mut args = args.iter().copied(); + while let Some(arg) = args.next() { + let value = if let Some(inline) = arg.strip_prefix("--engine=") { + Some(inline) + } else if arg == "--engine" { + args.next() + } else { + continue; + }; + return value.filter(|engine| !engine.is_empty()); + } + None +} + +/// What a failed serve's report says when no engine could be determined, so a +/// skipped backend inventory is visible rather than silently absent. +pub const ENGINE_UNKNOWN_EVIDENCE: &str = "engine unknown: backend evidence skipped"; + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -120,14 +155,14 @@ fn slugify(name: &str) -> String { /// /// The log may still be being written, so `len` is a snapshot: the head and tail /// are always real, and only the elided count can be slightly stale. -fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { +pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; let mut file = std::fs::File::open(path)?; let len = file.metadata()?.len(); if len <= max { let mut body = Vec::new(); - file.read_to_end(&mut body)?; + file.take(max).read_to_end(&mut body)?; return Ok((body, String::new())); } // `len > max >= 2 * half`, so the two halves can never overlap. @@ -230,6 +265,8 @@ pub struct ServeAttempt<'a> { /// failed, means the engine may still hold the port and the device — which /// changes how every later failure in the run should be read. pub stop_status: &'a str, + /// Only Lemonade attempts carry the optional backend inventory. + pub backend_evidence: Option<&'a str>, } /// Render one serve attempt's evidence as a failure message. @@ -241,7 +278,7 @@ pub struct ServeAttempt<'a> { #[must_use] pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { use crate::section; - format!( + let mut report = format!( "{}\n{}\n{}\n{}\n{}\n{}\n{}", attempt.headline, section("device state", attempt.device_state), @@ -250,14 +287,20 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { section("service log (tail)", attempt.log_tail), section("service log (full)", attempt.archived_log), section("stop", attempt.stop_status), - ) + ); + if let Some(evidence) = attempt.backend_evidence { + report.push_str("\n--- Lemonade backend evidence ---\n"); + report.push_str(evidence); + } + report } #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - serve_attempt_report, service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, + parse_log_path, read_clamped, reported_engine, serve_attempt_report, service_log_tail, + slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -270,6 +313,36 @@ managed service launched readiness: starting "; + #[test] + fn engine_lookup_is_safe_for_missing_or_incomplete_serve_plans() { + assert_eq!(reported_engine(SERVE_PLAN), Some("vllm")); + assert_eq!(reported_engine(" engine: lemonade\n"), Some("lemonade")); + for output in ["", "serve failed", " engine: \n"] { + assert_eq!(reported_engine(output), None); + } + } + + #[test] + fn engine_is_read_from_both_flag_forms_and_absent_otherwise() { + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine", "lemonade", "--managed"]), + Some("lemonade") + ); + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine=lemonade", "--managed"]), + Some("lemonade") + ); + for args in [ + &["serve", "m", "--managed"][..], + &["serve", "m", "--engine"], + &["serve", "m", "--engine="], + &["serve", "m", "--engine", ""], + &["serve", "--engine-version=1"], + ] { + assert_eq!(engine_from_serve_args(args), None, "{args:?}"); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( @@ -502,7 +575,7 @@ managed service launched /// full log went — none of which the plain readiness timeout reported. #[test] fn a_stalled_serve_report_carries_every_piece_of_evidence() { - let report = serve_attempt_report(&ServeAttempt { + let mut attempt = ServeAttempt { headline: "`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed` \ exited 0, but the endpoint never served `Qwen3-0.6B` within 600s", device_state: "device state: drained (48000 MiB free of 49000 MiB, floor 44100 MiB)", @@ -511,7 +584,15 @@ managed service launched log_tail: "load: failed to load model", archived_log: "archived to service-logs/scenario--svc.log", stop_status: "lemonade-qwen: stopped", - }); + backend_evidence: None, + }; + let report = serve_attempt_report(&attempt); + assert!(!report.contains("Lemonade backend evidence"), "{report}"); + attempt.backend_evidence = Some("fixture backend identity"); + assert_eq!( + serve_attempt_report(&attempt), + format!("{report}\n--- Lemonade backend evidence ---\nfixture backend identity") + ); for expected in [ "exited 0", diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 455a5ae46d..129a0a3795 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,7 +13,8 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, + ENGINE_UNKNOWN_EVIDENCE, ServeAttempt, archive_service_log, engine_from_serve_args, + reported_engine, serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -139,8 +140,9 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( - world: &E2eWorld, +async fn serve_failure_evidence( + world: &mut E2eWorld, + engine: Option<&str>, headline: &str, device_state: &str, stdout: &str, @@ -155,6 +157,19 @@ fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); + let backend_evidence = match engine { + Some("lemonade") => Some( + e2e_cucumber::lemonade_diagnostics::backend_evidence( + &world + .app_paths() + .engine_manifests_dir("lemonade") + .join("runtime.json"), + ) + .await, + ), + Some(_) => None, + None => Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()), + }; let stop_status = stop_scenario_services(world); serve_attempt_report(&ServeAttempt { headline, @@ -164,6 +179,7 @@ fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, + backend_evidence: backend_evidence.as_deref(), }) } @@ -202,7 +218,15 @@ async fn serve_and_wait( }; panic!( "{}", - serve_failure_evidence(world, &headline, &device_state, &stdout, &stderr) + serve_failure_evidence( + world, + engine_from_serve_args(args).or_else(|| reported_engine(&stdout)), + &headline, + &device_state, + &stdout, + &stderr, + ) + .await ); } @@ -609,13 +633,17 @@ async fn serve_gpu_model(world: &mut E2eWorld) { // outcome is quoted: a stop that found no record (the launch had not yet // written one) or failed means the next attempt did NOT get a clean // device, and the failure says so instead of looking like a broken serve. - diagnostics.push(serve_failure_evidence( - world, - &format!("attempt {attempt} of {attempts} (rc={rc})"), - &device_state, - &stdout, - &stderr, - )); + diagnostics.push( + serve_failure_evidence( + world, + Some(engine), + &format!("attempt {attempt} of {attempts} (rc={rc})"), + &device_state, + &stdout, + &stderr, + ) + .await, + ); if attempt == attempts { break; } @@ -992,15 +1020,19 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { &ready_substr, timeout_secs, ); + let stdout = world.cli_output.clone().unwrap_or_default(); + let stderr = world.cli_stderr.clone().unwrap_or_default(); panic!( "{}", serve_failure_evidence( world, + reported_engine(&stdout), &headline, &device_state, - world.cli_output.as_deref().unwrap_or_default(), - world.cli_stderr.as_deref().unwrap_or_default(), + &stdout, + &stderr, ) + .await ); } } @@ -2453,11 +2485,7 @@ async fn assert_endpoint_port(world: &mut E2eWorld) { /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { - let Some(engine) = output - .lines() - .find_map(|l| l.trim().strip_prefix("engine:")) - .map(str::trim) - else { + let Some(engine) = reported_engine(output) else { panic!("no 'engine:' line in serve output:\n{output}"); }; engine From f4755923ddfbc19907244aa1cc4088fcc5495ad1 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:26:33 +0000 Subject: [PATCH 02/28] test(e2e): treat a flag after --engine as no engine in serve failure reports A serve invocation such as 'serve m --engine --managed' made the engine lookup return '--managed'. The failure report then skipped the backend inventory without printing the 'engine unknown' line that makes a skipped inventory visible. A value that starts with '--' is another flag, not an engine, so it now counts as absent, for both the separate and the '--engine=' forms. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/serve_log.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index f0aff72532..a9734d1cd9 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -69,7 +69,10 @@ pub fn reported_engine(serve_stdout: &str) -> Option<&str> { /// /// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: /// the CLI then picks a default, which only the printed plan reveals (see -/// [`reported_engine`]). +/// [`reported_engine`]). A value that starts with `--` is another flag, not an +/// engine (`serve m --engine --managed`), so it counts as absent too: reporting +/// `--managed` as the engine would skip the backend inventory without the +/// [`ENGINE_UNKNOWN_EVIDENCE`] line that makes the skip visible. #[must_use] pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { let mut args = args.iter().copied(); @@ -81,7 +84,7 @@ pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { } else { continue; }; - return value.filter(|engine| !engine.is_empty()); + return value.filter(|engine| !engine.is_empty() && !engine.starts_with("--")); } None } @@ -337,6 +340,8 @@ managed service launched &["serve", "m", "--engine"], &["serve", "m", "--engine="], &["serve", "m", "--engine", ""], + &["serve", "m", "--engine", "--managed"], + &["serve", "m", "--engine=--managed"], &["serve", "--engine-version=1"], ] { assert_eq!(engine_from_serve_args(args), None, "{args:?}"); From f237d2d9bd8b25a167b1585de932ec171be1e5db Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:30:16 +0000 Subject: [PATCH 03/28] test(e2e): keep the per-row flaky rule in the expectations header The header comment replaced in this PR also said that flaky is set per row and is not derived from the bug reference, so a new row for the same bug has to set it explicitly. Restore that sentence next to the rule that keeps flaky=true for the #897 rows. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/expectations.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 3acdaf3533..d06138e9e9 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -64,8 +64,10 @@ reason = "A vLLM serve of a short name does not resolve to lemonade's full model # (41986772728). A backend ready line does not prove a later shutdown. # EAI-7423 described only the earlier ready-then-unavailable symptom. # Retain flaky=true because historical runs also passed intermittently; do not -# infer a fix from a single XPASS. Failed serves now capture candidate executable -# --version/--help and configured pins separately, before stopping the service. +# infer a fix from a single XPASS. The flag is per-row and is NOT derived from +# `bug`, so a new #897 row needs `flaky = true` added explicitly. Failed serves +# now capture candidate executable --version/--help and configured pins +# separately, before stopping the service. # # os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] From 26496115f21127b61083453aa0c3608b553048fe Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:31:19 +0000 Subject: [PATCH 04/28] test(e2e): cover Linux hosts without a usable GPU in the #897 expectation rows The existing regression test resolves the real-GPU rows against the mock capability, whose os_family is 'other', so the os=linux condition excluded it before the therock_family and has_amd_gpu guards were consulted. Removing either guard left the test green even though Linux MockServer-backed bench and chat scenarios would then be expected to fail. Add Linux hosts with no GPU, with a gfx target but no usable GPU, and with a GPU but no gfx target, and assert that a full real-GPU host is still expected to fail. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/expectation.rs | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 60c58f495f..c90e72aca9 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -859,6 +859,69 @@ mod tests { } } + /// The rows that carry `therock_family` and `has_amd_gpu` exist so that the + /// no-GPU lane, where the same scenarios run against MockServer instead of a + /// real Lemonade serve, stays expect-pass. `cap("mock")` cannot prove that: + /// its `os_family` is `other`, so the `os = "linux"` condition excludes it + /// before either GPU guard is consulted. These hosts are Linux, so each guard + /// is the only thing standing between them and an xfail. The two partial + /// hosts isolate the guards from each other: dropping `has_amd_gpu` from a row + /// fails the host that reports only a gfx target, and dropping `therock_family` + /// fails the host that reports only a GPU. + #[test] + fn real_gpu_lemonade_rows_do_not_xfail_linux_hosts_without_a_usable_gpu() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + let linux_lemonade = cap("strix-ubuntu"); + let hosts = [ + ( + "linux, no GPU, no gfx target", + HostCapability { + gfx_target: None, + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, gfx target reported but no usable GPU", + HostCapability { + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, usable GPU but no gfx target reported", + HostCapability { + gfx_target: None, + ..linux_lemonade.clone() + }, + ), + ]; + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + ] { + let scenario = decl(&[&format!("id:{id}")]); + for (label, host) in &hosts { + assert_eq!( + resolve(&scenario, host, &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {label}", + ); + } + // The full real-GPU host is the one that must still be xfailed. + assert!( + matches!( + resolve(&scenario, &linux_lemonade, &matrix, Included::default()), + Expectation::ExpectXfail { bug, .. } if bug == "#897" + ), + "{id} on a real GPU host", + ); + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } From 25e11c518039ece42387fef50868ae54d21db38e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:32:32 +0000 Subject: [PATCH 05/28] test(e2e): assert the report says why an oversized backend has no fingerprint The size-cap tests only covered the fingerprint helper. Cover the report itself: a llama-server larger than the hash cap is still listed and probed, and the report prints the reason it has no sha256 line, so the missing fingerprint is visible in the failure output. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 0dfcddd7e7..f28dbc81cd 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -393,6 +393,32 @@ mod tests { assert!(!report.contains(&skipped.display().to_string()), "{report}"); } + /// An executable past the hash cap is not fingerprinted, but the report says + /// so instead of omitting the line, and still lists the candidate and probes it. + #[tokio::test] + async fn oversized_executable_reports_why_it_has_no_fingerprint() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::File::create(&server) + .unwrap() + .set_len(MAX_HASH_BYTES + 1) + .unwrap(); + let report = backend_evidence(&write_manifest(root.path())).await; + assert!( + report.contains("sha256 unavailable: executable exceeds 32 MiB hash limit"), + "{report}" + ); + assert!(!report.contains("sha256: "), "{report}"); + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("--version:"), "{report}"); + } + #[tokio::test] async fn exhausted_budget_skips_hashing_and_probes() { let root = tempfile::tempdir().unwrap(); From 5f37981a6d02ce9cc2fad425c201e2afa020c9bf Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 15:45:35 +0000 Subject: [PATCH 06/28] docs(testing): state the flag-after---engine, oversized-binary and probe-timing behaviour Document that a value starting with -- after --engine counts as no engine named, that a llama-server over the hash cap is listed and probed with an explicit 'sha256 unavailable' reason, and that the version/help probes run before the failed service is stopped, with the GPU contention question marked as unchecked. Signed-off-by: Jussi Elo --- docs/testing.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index b5fec92dca..cd0d6b23a1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -820,7 +820,9 @@ relying on this coverage. On a failed serve, the shared serve steps preserve the full service log under `service-logs/` in the E2E results artifact before stopping the service. The engine is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if -neither names one, the report says `engine unknown: backend evidence skipped`. +neither names one, the report says `engine unknown: backend evidence skipped`. A +value that starts with `--` (as in `serve m --engine --managed`) is another flag, not +an engine, and counts as no engine named. Lemonade failures also include the installed runtime manifest's version/backend, configured `backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints and `version.txt`, @@ -831,14 +833,22 @@ marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). Each probe has a five-second timeout. Hashing and probes share a 20-second budget after inventory discovery; hashing runs off the async executor, rejects files -over 32 MiB, and checks its deadline between 64 KiB reads. Filesystem operations -and process cleanup are not preemptible, so this is not a hard wall-clock limit -on the entire report. +over 32 MiB, and checks its deadline between 64 KiB reads. A candidate over the +size cap gets no fingerprint, and its entry says so with `sha256 unavailable: +executable exceeds 32 MiB hash limit`; it is still listed and probed. Filesystem +operations and process cleanup are not preemptible, so this is not a hard +wall-clock limit on the entire report. A candidate or configured build pin is **not proof** of which binary the router launched; a failed probe is retained as evidence, not treated as an identity. Probes inherit the harness environment rather than reconstructing the engine's managed library paths, so a probe loader error alone does not establish why the -serve failed. +serve failed. The probes run once the serve has exited non-zero or its readiness +wait has timed out, and after the log has been read and archived, but before the +service is stopped, so the router may still be running while they execute. Whether +a `llama-server --version` or `--help` run initialises a GPU backend, and so could +contend with that router on a shared-memory device, has not been checked; the +probes are bounded and sequential, and the attempt's failing outcome is already +decided when they start. Use the evidence to distinguish backend startup, download, and router readiness failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the From c05b62ceed60386347d4e266d1ceef46e9cdb4fc Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 16:01:50 +0000 Subject: [PATCH 07/28] test(e2e): compare backend candidate paths as paths, not printed text The oversized-executable test asserted that the report contained the display form of a path it built with join("bin/llamacpp/rocm-stable"), while the inventory joins "bin/llamacpp" first and the backend directory second. On Windows the two spell the same path with different separators, so the substring check failed even though the report was right. Parse the candidate lines back into paths and compare those; Path equality is component-wise, so both spellings agree. The same helper replaces a negative check in the candidate-cap test that could never fail on Windows: asserting the report did not contain the display form of the skipped candidate passed vacuously there. It now asserts the listed candidates exactly (count, the skipped one absent, the first one present). Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index f28dbc81cd..b446f03f74 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -297,6 +297,22 @@ mod tests { manifest } + /// The candidate paths a report lists, parsed back into paths. + /// + /// Compared as paths, never as printed text: the inventory joins + /// `bin/llamacpp` as one string, so on Windows a report line mixes `/` and + /// `\` while a test that builds the expected path another way prints it + /// differently. `Path` equality compares components, so both spellings agree, + /// and a negative check ("this candidate was skipped") cannot pass vacuously + /// just because the two spellings differ. + fn candidate_paths(report: &str) -> Vec { + report + .lines() + .filter_map(|line| line.strip_prefix("candidate (not proof of selection): ")) + .map(PathBuf::from) + .collect() + } + #[test] fn executable_identity_does_not_depend_on_the_dynamic_loader() { let root = tempfile::tempdir().unwrap(); @@ -390,7 +406,16 @@ mod tests { let skipped = root .path() .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); - assert!(!report.contains(&skipped.display().to_string()), "{report}"); + let listed = candidate_paths(&report); + assert_eq!(listed.len(), MAX_CANDIDATES, "{report}"); + assert!(!listed.contains(&skipped), "{report}"); + let first = root + .path() + .join(format!("bin/llamacpp/rocm-stable/00/{binary}")); + assert!( + listed.contains(&first), + "the comparison must be able to match a listed candidate: {report}" + ); } /// An executable past the hash cap is not fingerprinted, but the report says @@ -415,7 +440,7 @@ mod tests { "{report}" ); assert!(!report.contains("sha256: "), "{report}"); - assert!(report.contains(&server.display().to_string()), "{report}"); + assert_eq!(candidate_paths(&report), vec![server], "{report}"); assert!(report.contains("--version:"), "{report}"); } From 05bcc1c0ebd949e1f07b0accffca1c0ce2e410e7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 14:52:40 +0000 Subject: [PATCH 08/28] fix(e2e): read the Lemonade runtime manifest through the path the engine writes it to The serve failure report reads the installed runtime's manifest to find the backend directory, and built the path with its own "runtime.json" literal next to the engine's. A rename on either side would have degraded the report to "unavailable:" on the one lane it exists for, with nothing failing before then. AppPaths::engine_runtime_manifest_path is now the one definition. The engine's manifest_path and the harness both call it, and an engine test writes a manifest and asserts it lands at that path with the field names the report reads. Signed-off-by: Jussi Elo --- crates/rocm-core/src/paths.rs | 19 +++++++++ engines/lemonade/src/install.rs | 39 ++++++++++++++++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 5 +-- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 15840e56e9..7a2ff4d2c1 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -194,6 +194,16 @@ impl AppPaths { self.engine_dir(engine).join("manifests") } + /// The manifest an engine's installer writes for its installed runtime. + /// + /// One definition for the writer (the engine) and for every reader of the + /// file, so a reader cannot drift to a different name: a reader that + /// misses it degrades to "no manifest", which is indistinguishable from an + /// engine that was never installed. + pub fn engine_runtime_manifest_path(&self, engine: &str) -> PathBuf { + self.engine_manifests_dir(engine).join("runtime.json") + } + pub fn engine_state_dir(&self, engine: &str) -> PathBuf { self.engine_dir(engine).join("state") } @@ -355,6 +365,15 @@ mod tests { paths.data_dir.join("automations").join("proposals.jsonl") ); } + #[test] + fn engine_runtime_manifest_lives_in_the_engine_manifests_dir() { + let (_root, paths) = temp_app_paths("engine-runtime-manifest-path"); + assert_eq!( + paths.engine_runtime_manifest_path("lemonade"), + paths.engine_manifests_dir("lemonade").join("runtime.json") + ); + } + #[test] fn with_managed_root_keeps_reprovisioning_flat_from_runtime_leaf() { let data_root = PathBuf::from("/tmp/rocm-cli-reprovision"); diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index e0fa931d11..2412131787 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -196,7 +196,7 @@ where } fn manifest_path(paths: &AppPaths) -> PathBuf { - paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json") + paths.engine_runtime_manifest_path(ENGINE_NAME) } fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { @@ -573,6 +573,43 @@ mod tests { #[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; + /// The e2e harness reads this manifest back to describe a failed serve + /// (`tests/e2e-cucumber/src/lemonade_diagnostics.rs`), by the same path + /// helper and the field names asserted here. A rename on either side would + /// otherwise show up only as `unavailable:` in a report from the real lane. + #[test] + fn written_manifest_is_where_and_how_the_e2e_failure_report_reads_it() { + let dir = scratch_dir("manifest-path-and-fields"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let manifest = LemonadeInstallManifest { + env_id: "lemonade-embeddable-fixture".to_owned(), + version: "fixture".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("runtime/lemond"), + lemonade: dir.join("runtime/lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 1, + }; + write_manifest(&paths, &manifest).unwrap(); + + let path = paths.engine_runtime_manifest_path("lemonade"); + assert_eq!(manifest_path(&paths), path); + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(written["version"], "fixture"); + assert_eq!(written["backend_recipe"], LLAMACPP_RECIPE); + assert_eq!(written["backend_name"], ROCM_BACKEND_NAME); + assert_eq!( + written["runtime_dir"], + serde_json::json!(manifest.runtime_dir) + ); + fs::remove_dir_all(&dir).ok(); + } + #[test] fn embeddable_package_matches_runtime_os() { let version = rocm_deps::LEMONADE_VERSION; diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 129a0a3795..1f7e8a12ee 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -160,10 +160,7 @@ async fn serve_failure_evidence( let backend_evidence = match engine { Some("lemonade") => Some( e2e_cucumber::lemonade_diagnostics::backend_evidence( - &world - .app_paths() - .engine_manifests_dir("lemonade") - .join("runtime.json"), + &world.app_paths().engine_runtime_manifest_path("lemonade"), ) .await, ), From dfb844f0b75b7be068ba682cc3a914eb0c37d971 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:02:29 +0000 Subject: [PATCH 09/28] fix(e2e): run the llama-server probes after the failed service is stopped The version and help probes executed a backend binary while the failed serve's router could still be running. llama.cpp builds commonly initialise a GPU backend even for --version, so on a shared-memory device a probe could contend with the router and change the state the report exists to describe. docs/testing.md said this had not been checked. The report now reads files first and executes second: manifest, pins, candidate paths, fingerprints and version.txt are taken before the stop, and the probes run after it, with their own 20 second budget so a slow stop cannot use it up. The order lives in one function, backend_evidence_around_stop, which also runs the stop exactly once for every engine, so the serve step cannot reorder or skip it. Its tests make the stop delete the candidate, which shows the fingerprint came before the stop and, through a recording probe, that the probes came after it. docs/testing.md and the expectations header now state what the code does. Signed-off-by: Jussi Elo --- docs/testing.md | 26 +- tests/e2e-cucumber/expectations.toml | 4 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 342 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 27 +- 4 files changed, 332 insertions(+), 67 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index ea8c4873da..ed8b1d4d3e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -831,8 +831,9 @@ are hashed and probed, with two read-only probes per candidate. Each probe retai up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). -Each probe has a five-second timeout. Hashing and probes share a 20-second budget -after inventory discovery; hashing runs off the async executor, rejects files +Each probe has a five-second timeout. The inventory, including hashing, has a +20-second budget and the probes have their own 20-second budget, which starts +when the service stop returns; hashing runs off the async executor, rejects files over 32 MiB, and checks its deadline between 64 KiB reads. A candidate over the size cap gets no fingerprint, and its entry says so with `sha256 unavailable: executable exceeds 32 MiB hash limit`; it is still listed and probed. Filesystem @@ -842,13 +843,20 @@ A candidate or configured build pin is **not proof** of which binary the router launched; a failed probe is retained as evidence, not treated as an identity. Probes inherit the harness environment rather than reconstructing the engine's managed library paths, so a probe loader error alone does not establish why the -serve failed. The probes run once the serve has exited non-zero or its readiness -wait has timed out, and after the log has been read and archived, but before the -service is stopped, so the router may still be running while they execute. Whether -a `llama-server --version` or `--help` run initialises a GPU backend, and so could -contend with that router on a shared-memory device, has not been checked; the -probes are bounded and sequential, and the attempt's failing outcome is already -decided when they start. +serve failed. The report reads files first and executes a backend second. The +manifest, pins, candidate paths, fingerprints and `version.txt` are read once the serve has exited +non-zero or its readiness wait has timed out and the log has been read and +archived, but before the service is stopped, so they describe the state being +diagnosed. The `--version` and `--help` probes run after the stop returns, in the +report's "probes, run after the service stop was attempted" section: llama.cpp +builds commonly initialise a GPU backend even for `--version`, and doing that next +to a still-running router on a shared-memory device could change the state being +diagnosed. A stop that fails or finds no record can leave the router running; the +report's `stop` section says so, and the probes still run. No lane has shown +whether a probe initialises a GPU backend, which is why they are kept off the +router's side of the stop. The order is fixed in one function, +`backend_evidence_around_stop`, whose unit tests make the stop delete the candidate +to show the fingerprint was read before it and the probes came after it. Use the evidence to distinguish backend startup, download, and router readiness failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 0ce0a7bae5..85d5642773 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -66,8 +66,8 @@ reason = "A vLLM serve of a short name does not resolve to lemonade's full model # Retain flaky=true because historical runs also passed intermittently; do not # infer a fix from a single XPASS. The flag is per-row and is NOT derived from # `bug`, so a new #897 row needs `flaky = true` added explicitly. Failed serves -# now capture candidate executable --version/--help and configured pins -# separately, before stopping the service. +# now capture configured pins, candidate executable paths and fingerprints +# before stopping the service, and each candidate's --version/--help after it. # # os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index b446f03f74..128154d916 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -2,7 +2,13 @@ // // SPDX-License-Identifier: MIT -//! Read-only backend evidence for a failed Lemonade serve, before teardown. +//! Read-only backend evidence for a failed Lemonade serve. +//! +//! The evidence is taken in two steps with the service stop between them (see +//! [`backend_evidence_around_stop`]): what is only read from disk (manifest, +//! pins, candidate paths, fingerprints) is collected while the failed service is +//! still in the state being diagnosed, and the `llama-server` probes, which +//! execute a backend binary, run after it has been stopped. use sha2::{Digest as _, Sha256}; use std::fmt::Write as _; @@ -12,31 +18,147 @@ use std::process::Stdio; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncReadExt as _}; -use crate::serve_log::read_clamped; +use crate::serve_log::{ENGINE_UNKNOWN_EVIDENCE, read_clamped}; const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; const PROBE_TIMEOUT: Duration = Duration::from_secs(5); const MAX_CANDIDATES: usize = 8; -const TOTAL_EVIDENCE_BUDGET: Duration = Duration::from_secs(20); +/// Time for the read-only inventory (pins, candidate paths, fingerprints), +/// taken before the service is stopped. +const INVENTORY_BUDGET: Duration = Duration::from_secs(20); +/// Time for the probes, started after the stop returns so a slow stop does not +/// use it up. +const PROBE_BUDGET: Duration = Duration::from_secs(20); const MAX_HASH_BYTES: u64 = 32 * 1024 * 1024; +#[derive(Clone, Copy)] +struct Budgets { + inventory: Duration, + probes: Duration, +} + +const DEFAULT_BUDGETS: Budgets = Budgets { + inventory: INVENTORY_BUDGET, + probes: PROBE_BUDGET, +}; + +/// Runs one `llama-server` flag and renders what came back. A trait so the order +/// of probes against the service stop is testable without executing a binary. +trait Probe { + async fn run(&self, server: &Path, flag: &str, timeout: Duration) -> String; +} + +struct RealProbe; + +impl Probe for RealProbe { + async fn run(&self, server: &Path, flag: &str, timeout: Duration) -> String { + probe(server, flag, timeout).await + } +} + +/// Collect the backend evidence for a failed serve of `engine` around the call +/// that stops the service, and return the stop's own status with it. +/// +/// `stop` always runs exactly once, whatever the engine, so a caller cannot +/// forget it or run it twice. For Lemonade the order is fixed: the read-only +/// inventory first, then `stop`, then the probes. A probe executes a backend +/// binary, and llama.cpp builds commonly initialise a GPU backend even for +/// `--version`, so running one next to a still-live router on a shared-memory +/// device could change the very state being diagnosed. Reading files cannot. +/// `manifest_path` is only called for Lemonade. +/// +/// The report says `engine unknown` when no engine was determined, so a skipped +/// inventory stays visible, and returns no evidence for another engine. +pub async fn backend_evidence_around_stop( + manifest_path: impl FnOnce() -> PathBuf, + engine: Option<&str>, + stop: impl FnOnce() -> String, +) -> (String, Option) { + match engine { + Some("lemonade") => { + let (stop_status, evidence) = + evidence_around_stop(&manifest_path(), DEFAULT_BUDGETS, &RealProbe, stop).await; + (stop_status, Some(evidence)) + } + Some(_) => (stop(), None), + None => (stop(), Some(ENGINE_UNKNOWN_EVIDENCE.to_owned())), + } +} + +async fn evidence_around_stop( + manifest_path: &Path, + budgets: Budgets, + probe: &impl Probe, + stop: impl FnOnce() -> String, +) -> (String, String) { + let inventory = take_inventory(manifest_path, budgets.inventory).await; + let stop_status = stop(); + let probes = inventory.probe_all(probe, budgets.probes).await; + (stop_status, format!("{}{probes}", inventory.report)) +} + +/// What was read from disk before the stop: the report so far, and the +/// candidates the probes will run against once the service is stopped. +struct Inventory { + report: String, + servers: Vec, +} + +impl Inventory { + fn without_probes(report: String) -> Self { + Self { + report, + servers: Vec::new(), + } + } + + async fn probe_all(&self, probe: &impl Probe, budget: Duration) -> String { + if self.servers.is_empty() { + return String::new(); + } + let mut report = String::new(); + let _ = writeln!( + report, + "probes, run after the service stop was attempted; they inherit the harness \ + environment, not the engine's managed library environment, so a probe loader \ + failure does not establish a serving failure:" + ); + let deadline = Instant::now() + budget; + for server in &self.servers { + let _ = writeln!(report, "probe target: {}", server.display()); + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let output = probe.run(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + report + } +} + /// Record the installed version and pins separately from executable build output: /// a pin describes what should be installed, not what a runner actually cached. /// /// Every candidate is labelled as such; finding a binary does not prove that the -/// router launched it. No install, download, model load, or cache repair is done. -pub async fn backend_evidence(manifest_path: &Path) -> String { - backend_evidence_with_budget(manifest_path, TOTAL_EVIDENCE_BUDGET).await -} - -async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> String { +/// router launched it. Nothing here executes anything. No install, download, +/// model load, or cache repair is done. +async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { let mut report = String::new(); let _ = writeln!(report, "manifest: {}", manifest_path.display()); let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { serde_json::from_slice::(&bytes).map_err(std::io::Error::other) }) { Ok(manifest) => manifest, - Err(error) => return format!("{report}unavailable: {error}"), + Err(error) => { + return Inventory::without_probes(format!("{report}unavailable: {error}")); + } }; for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { let _ = writeln!( @@ -49,7 +171,9 @@ async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> .get("runtime_dir") .and_then(serde_json::Value::as_str) else { - return format!("{report}backend inventory unavailable: no runtime_dir"); + return Inventory::without_probes(format!( + "{report}backend inventory unavailable: no runtime_dir" + )); }; let runtime = Path::new(runtime); let pins = runtime.join("resources/backend_versions.json"); @@ -63,11 +187,6 @@ async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> } } let candidates = backend_candidates(runtime); - let _ = writeln!( - report, - "probes inherit the harness environment, not the engine's managed library environment; \ - a probe loader failure does not establish a serving failure" - ); if candidates.is_empty() { let _ = writeln!( report, @@ -75,19 +194,21 @@ async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> ); } let deadline = Instant::now() + budget; + let mut servers = Vec::new(); for server in candidates.iter().take(MAX_CANDIDATES) { if Instant::now() >= deadline { let _ = writeln!( report, "remaining evidence skipped: evidence budget exhausted" ); - return report; + return Inventory { report, servers }; } let _ = writeln!( report, "candidate (not proof of selection): {}", server.display() ); + servers.push(server.clone()); let path = server.clone(); match tokio::task::spawn_blocking(move || fingerprint(&path, deadline)) .await @@ -112,18 +233,6 @@ async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> ); } } - for flag in ["--version", "--help"] { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - let _ = writeln!( - report, - "remaining evidence skipped: evidence budget exhausted" - ); - return report; - } - let output = probe(server, flag, remaining.min(PROBE_TIMEOUT)).await; - let _ = writeln!(report, "{flag}:\n{output}"); - } } if candidates.len() > MAX_CANDIDATES { let _ = writeln!( @@ -131,7 +240,7 @@ async fn backend_evidence_with_budget(manifest_path: &Path, budget: Duration) -> "backend inventory truncated after {MAX_CANDIDATES} candidates" ); } - report + Inventory { report, servers } } fn backend_candidates(runtime: &Path) -> Vec { @@ -287,6 +396,16 @@ async fn drain_output( mod tests { use super::*; + async fn evidence(manifest: &Path) -> String { + evidence_with(manifest, DEFAULT_BUDGETS).await + } + + async fn evidence_with(manifest: &Path, budgets: Budgets) -> String { + evidence_around_stop(manifest, budgets, &RealProbe, String::new) + .await + .1 + } + fn write_manifest(runtime: &Path) -> PathBuf { let manifest = runtime.join("runtime.json"); std::fs::write( @@ -319,7 +438,7 @@ mod tests { let server = root.path().join("not-executable"); std::fs::write(&server, b"abc").unwrap(); assert_eq!( - fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET).unwrap(), + fingerprint(&server, Instant::now() + INVENTORY_BUDGET).unwrap(), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" ); } @@ -331,14 +450,14 @@ mod tests { let file = std::fs::File::create(&server).unwrap(); file.set_len(MAX_HASH_BYTES).unwrap(); assert_eq!( - fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + fingerprint(&server, Instant::now() + INVENTORY_BUDGET) .unwrap() .len(), 64 ); file.set_len(MAX_HASH_BYTES + 1).unwrap(); assert_eq!( - fingerprint(&server, Instant::now() + TOTAL_EVIDENCE_BUDGET) + fingerprint(&server, Instant::now() + INVENTORY_BUDGET) .unwrap_err() .to_string(), "executable exceeds 32 MiB hash limit" @@ -392,7 +511,7 @@ mod tests { std::fs::create_dir_all(server.parent().unwrap()).unwrap(); std::fs::write(server, "").unwrap(); } - let report = backend_evidence(&write_manifest(root.path())).await; + let report = evidence(&write_manifest(root.path())).await; assert_eq!( report .matches("candidate (not proof of selection):") @@ -434,7 +553,7 @@ mod tests { .unwrap() .set_len(MAX_HASH_BYTES + 1) .unwrap(); - let report = backend_evidence(&write_manifest(root.path())).await; + let report = evidence(&write_manifest(root.path())).await; assert!( report.contains("sha256 unavailable: executable exceeds 32 MiB hash limit"), "{report}" @@ -455,14 +574,146 @@ mod tests { let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); std::fs::create_dir_all(server.parent().unwrap()).unwrap(); std::fs::write(server, "").unwrap(); - let report = - backend_evidence_with_budget(&write_manifest(root.path()), Duration::ZERO).await; + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: Duration::ZERO, + probes: PROBE_BUDGET, + }, + ) + .await; assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); for skipped in ["sha256", "--version:", "--help:", "candidate ("] { assert!(!report.contains(skipped), "{report}"); } } + struct RecordingProbe<'a> { + events: &'a std::cell::RefCell>, + } + + impl Probe for RecordingProbe<'_> { + async fn run(&self, _server: &Path, flag: &str, _timeout: Duration) -> String { + self.events.borrow_mut().push(format!("probe {flag}")); + "recorded, not executed".to_owned() + } + } + + /// A lemonade data dir whose manifest sits where the engine writes it, with one + /// small candidate whose SHA-256 is a known value. + fn lemonade_data_dir_with_candidate() -> (tempfile::TempDir, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let paths = rocm_core::AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let runtime = root.path().join("runtime"); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = runtime.join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, b"abc").unwrap(); + let manifest = paths.engine_runtime_manifest_path("lemonade"); + std::fs::create_dir_all(manifest.parent().unwrap()).unwrap(); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + (root, manifest, server) + } + + const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + + /// The inventory is read while the failed service is still in place and the + /// probes, which execute a backend binary, run only after the stop. + /// + /// The stop deletes the candidate. A fingerprint in the report therefore proves + /// the file was read before the stop, and the recorded events prove the probes + /// came after it; running either step on the wrong side turns this red. + #[tokio::test] + async fn inventory_precedes_the_stop_and_probes_follow_it() { + let (_root, manifest, server) = lemonade_data_dir_with_candidate(); + let events = std::cell::RefCell::new(Vec::new()); + let (stop_status, report) = evidence_around_stop( + &manifest, + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || { + events.borrow_mut().push("stop".to_owned()); + std::fs::remove_file(&server).unwrap(); + "stopped".to_owned() + }, + ) + .await; + assert_eq!(stop_status, "stopped"); + assert_eq!( + *events.borrow(), + ["stop", "probe --version", "probe --help"], + "{report}" + ); + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!(!report.contains("sha256 unavailable"), "{report}"); + let inventory = report.find("candidate (not proof of selection):").unwrap(); + let probes = report.find("probe target:").unwrap(); + assert!(inventory < probes, "{report}"); + assert!(report.contains("recorded, not executed"), "{report}"); + } + + #[tokio::test] + async fn stop_runs_once_for_every_engine_and_an_unknown_engine_stays_visible() { + let stops = std::cell::Cell::new(0); + let stop = || { + stops.set(stops.get() + 1); + "stopped".to_owned() + }; + let no_manifest = || -> PathBuf { panic!("the manifest is only read for lemonade") }; + + assert_eq!( + backend_evidence_around_stop(no_manifest, Some("vllm"), stop).await, + ("stopped".to_owned(), None) + ); + assert_eq!( + backend_evidence_around_stop(no_manifest, None, stop).await, + ( + "stopped".to_owned(), + Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()) + ) + ); + assert_eq!(stops.get(), 2); + + let (_root, manifest, _server) = lemonade_data_dir_with_candidate(); + let (status, evidence) = + backend_evidence_around_stop(|| manifest, Some("lemonade"), stop).await; + assert_eq!(status, "stopped"); + assert!(evidence.unwrap().contains("manifest: ")); + assert_eq!(stops.get(), 3); + } + + #[tokio::test] + async fn an_unreadable_manifest_is_reported_and_the_service_is_still_stopped() { + let root = tempfile::tempdir().unwrap(); + let events = std::cell::RefCell::new(Vec::new()); + let (stop_status, report) = evidence_around_stop( + &root.path().join("missing.json"), + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || "stopped".to_owned(), + ) + .await; + assert_eq!(stop_status, "stopped"); + assert!(report.contains("unavailable:"), "{report}"); + assert!(events.borrow().is_empty(), "no candidate, no probe"); + assert!(!report.contains("probe target:"), "{report}"); + } + #[cfg(unix)] #[tokio::test] async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { @@ -474,9 +725,14 @@ mod tests { "#!/bin/sh\nexec /bin/sleep 30\n", ) .unwrap(); - let report = - backend_evidence_with_budget(&write_manifest(root.path()), Duration::from_secs(1)) - .await; + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: INVENTORY_BUDGET, + probes: Duration::from_secs(1), + }, + ) + .await; assert!(report.contains("--version:\nprobe timed out"), "{report}"); assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); assert!(!report.contains("--help:"), "{report}"); @@ -527,7 +783,7 @@ mod tests { #[tokio::test] async fn missing_manifest_is_reported_without_panicking() { let root = tempfile::tempdir().unwrap(); - let report = backend_evidence(&root.path().join("missing.json")).await; + let report = evidence(&root.path().join("missing.json")).await; assert!(report.contains("unavailable:"), "{report}"); } @@ -552,7 +808,7 @@ mod tests { .to_string(), ) .unwrap(); - let report = backend_evidence(&manifest).await; + let report = evidence(&manifest).await; assert!( report.contains("configured pins (not executable identity):"), "{report}" @@ -600,7 +856,7 @@ esac .to_string(), ) .unwrap(); - let report = backend_evidence(&manifest).await; + let report = evidence(&manifest).await; assert!(report.contains(&server.display().to_string()), "{report}"); assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); assert!(report.contains("version.txt: cached-build"), "{report}"); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 1f7e8a12ee..936a3edd78 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,8 +13,8 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ENGINE_UNKNOWN_EVIDENCE, ServeAttempt, archive_service_log, engine_from_serve_args, - reported_engine, serve_attempt_report, service_log_tail, + ServeAttempt, archive_service_log, engine_from_serve_args, reported_engine, + serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -137,6 +137,9 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// rather than three calls at each site: the log must be read and copied out /// while the service still owns it, because stopping the service both changes /// what the log's last lines say and is the step most likely to fail outright. +/// For Lemonade the backend inventory is read before the stop too, and the +/// `llama-server` probes, which execute a backend binary, run only after it so +/// they cannot contend with the still-running router. /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. @@ -157,17 +160,15 @@ async fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); - let backend_evidence = match engine { - Some("lemonade") => Some( - e2e_cucumber::lemonade_diagnostics::backend_evidence( - &world.app_paths().engine_runtime_manifest_path("lemonade"), - ) - .await, - ), - Some(_) => None, - None => Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()), - }; - let stop_status = stop_scenario_services(world); + // The stop sits inside this call because its position relative to the + // backend probes is the point: see `backend_evidence_around_stop`. + let (stop_status, backend_evidence) = + e2e_cucumber::lemonade_diagnostics::backend_evidence_around_stop( + || world.app_paths().engine_runtime_manifest_path("lemonade"), + engine, + || stop_scenario_services(world), + ) + .await; serve_attempt_report(&ServeAttempt { headline, device_state, From 59d78919f5d662a9532b7f9d70f06fc0349451fa Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:04:44 +0000 Subject: [PATCH 10/28] fix(e2e): search deeper for llama-server candidates and say where an empty search looked The inventory looked only at depth 0 and 1 under four backend directories, while the router chooses how a llama.cpp build is unpacked, so a build one level deeper would have produced no candidates and no probe. When nothing was found, the report also suggested the backend download may not have completed, which an unsearched layout or a never-installed build would equally explain. The search now goes three directory levels below each backend directory, bounded by an entry cap per directory. An empty result names every location searched, the depth, that the cpu backend is deliberately left out, and what bin/llamacpp actually contains, without naming a cause. docs/testing.md states the search. Signed-off-by: Jussi Elo --- docs/testing.md | 8 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 169 +++++++++++++++--- 2 files changed, 154 insertions(+), 23 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index ed8b1d4d3e..3bb4c55b77 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -826,7 +826,13 @@ an engine, and counts as no engine named. Lemonade failures also include the installed runtime manifest's version/backend, configured `backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints and `version.txt`, -and each candidate's `--version` and `--help` output. At most eight candidates +and each candidate's `--version` and `--help` output. Candidates are searched +under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of +`bin/llamacpp`, down to three directory levels below each; the cpu backend is not +searched. When nothing is found, the report lists those locations and what +`bin/llamacpp` holds, and does not guess at the cause: an interrupted download, a +layout outside the search and a build that was never installed look the same. +At most eight candidates are hashed and probed, with two read-only probes per candidate. Each probe retains up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 128154d916..84993f26a4 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -188,10 +188,7 @@ async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { } let candidates = backend_candidates(runtime); if candidates.is_empty() { - let _ = writeln!( - report, - "no llama-server candidates under bin/llamacpp (backend download may not have completed)" - ); + let _ = writeln!(report, "{}", empty_search_note(runtime)); } let deadline = Instant::now() + budget; let mut servers = Vec::new(); @@ -243,31 +240,104 @@ async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { Inventory { report, servers } } -fn backend_candidates(runtime: &Path) -> Vec { - let binary = if cfg!(windows) { +/// The GPU backend directories under `bin/llamacpp` that are searched. The cpu +/// backend is left out on purpose: a CPU build is never what a GPU serve should +/// have launched, and listing it would read as one. +const BACKEND_DIRS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; +/// How many directory levels below a backend directory are searched. Lemonade +/// unpacks some llama.cpp builds into a build-numbered directory, and the layout +/// is the router's to choose, so the search is not pinned to one level. +const MAX_SEARCH_DEPTH: usize = 3; +/// Directory entries read per directory, so a corrupt tree cannot make the +/// search unbounded. +const MAX_DIR_ENTRIES: usize = 256; +/// Names listed when explaining an empty search. +const MAX_LISTED_NAMES: usize = 16; + +fn server_binary() -> &'static str { + if cfg!(windows) { "llama-server.exe" } else { "llama-server" - }; + } +} + +fn backend_candidates(runtime: &Path) -> Vec { let mut candidates = Vec::new(); - for backend in ["rocm-stable", "rocm-nightly", "rocm", "vulkan"] { - let dir = runtime.join("bin/llamacpp").join(backend); - let direct = dir.join(binary); - if direct.is_file() { - candidates.push(direct); - } - // Lemonade archives also unpack one level below the backend directory. - if let Ok(entries) = std::fs::read_dir(dir) { - let mut nested: Vec<_> = entries + for backend in BACKEND_DIRS { + collect_candidates( + &runtime.join("bin/llamacpp").join(backend), + 0, + &mut candidates, + ); + } + candidates +} + +fn collect_candidates(dir: &Path, depth: usize, candidates: &mut Vec) { + let direct = dir.join(server_binary()); + if direct.is_file() { + candidates.push(direct); + } + if depth >= MAX_SEARCH_DEPTH { + return; + } + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut subdirs: Vec = entries + .filter_map(Result::ok) + .take(MAX_DIR_ENTRIES) + .map(|entry| entry.path()) + .filter(|path| path.is_dir()) + .collect(); + subdirs.sort(); + for subdir in subdirs { + collect_candidates(&subdir, depth + 1, candidates); + } +} + +/// Why a search found nothing, in terms of where it looked. An absent binary can +/// mean a backend download that did not finish, a layout the search does not +/// cover, or a build that was never installed, and the report cannot tell which, +/// so it says what was searched and what `bin/llamacpp` actually holds. +fn empty_search_note(runtime: &Path) -> String { + let llamacpp = runtime.join("bin/llamacpp"); + let searched = BACKEND_DIRS + .iter() + .map(|backend| llamacpp.join(backend).display().to_string()) + .collect::>() + .join(", "); + let present = match std::fs::read_dir(&llamacpp) { + Ok(entries) => { + let mut names: Vec = entries .filter_map(Result::ok) - .map(|entry| entry.path().join(binary)) - .filter(|path| path.is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) .collect(); - nested.sort(); - candidates.extend(nested); + names.sort(); + if names.is_empty() { + format!("{} is empty", llamacpp.display()) + } else { + let more = names.len().saturating_sub(MAX_LISTED_NAMES); + names.truncate(MAX_LISTED_NAMES); + let more = if more > 0 { + format!(" and {more} more") + } else { + String::new() + }; + format!( + "{} contains: {}{more}", + llamacpp.display(), + names.join(", ") + ) + } } - } - candidates + Err(error) => format!("{} cannot be read: {error}", llamacpp.display()), + }; + format!( + "no llama-server candidates found; searched {searched}, down to {MAX_SEARCH_DEPTH} \ + directory levels below each (the cpu backend is not searched); {present}" + ) } fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { @@ -496,6 +566,61 @@ mod tests { assert_eq!(backend_candidates(root.path()), wanted); } + #[test] + fn candidate_search_reaches_three_levels_below_a_backend_and_no_further() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-nightly"); + // Literal layouts, not derived from MAX_SEARCH_DEPTH: the depth is the + // behaviour under test, so changing the constant must turn this red. + let found_at_three = backend.join("a/b/llama-b9752").join(server_binary()); + let found_at_two = backend.join("a/build").join(server_binary()); + let beyond = backend.join("a/b/c/llama-b9752").join(server_binary()); + for server in [&found_at_three, &found_at_two, &beyond] { + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + assert_eq!( + backend_candidates(root.path()), + vec![found_at_two, found_at_three] + ); + } + + /// An empty search says where it looked and what `bin/llamacpp` holds, and does + /// not guess that a download was interrupted: the same report follows from a + /// layout the search does not cover or a build that was never installed. + #[tokio::test] + async fn empty_search_lists_the_searched_locations_and_blames_nothing() { + let root = tempfile::tempdir().unwrap(); + let cpu = root.path().join("bin/llamacpp/cpu"); + std::fs::create_dir_all(&cpu).unwrap(); + std::fs::write(cpu.join(server_binary()), "").unwrap(); + let report = evidence(&write_manifest(root.path())).await; + let llamacpp = root.path().join("bin/llamacpp"); + for backend in BACKEND_DIRS { + assert!( + report.contains(&llamacpp.join(backend).display().to_string()), + "{report}" + ); + } + assert!( + report.contains("no llama-server candidates found"), + "{report}" + ); + assert!( + report.contains(&format!("{} contains: cpu", llamacpp.display())), + "{report}" + ); + assert!( + !report.contains("download") && !report.contains("complete"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + + let bare = tempfile::tempdir().unwrap(); + let report = evidence(&write_manifest(bare.path())).await; + assert!(report.contains("cannot be read"), "{report}"); + } + #[tokio::test] async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { let root = tempfile::tempdir().unwrap(); From f5bff5ff487d88b8260ce1d5c445e581577e169b Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:06:20 +0000 Subject: [PATCH 11/28] test(e2e): expect the depth-limit candidates in the order the search sorts them The previous commit's depth test listed its two expected candidates in the wrong order: the search sorts sibling directories by path, so a/b comes before a/build. The test therefore failed even with the depth limit in place. With the order corrected it passes, and with the limit lowered to two it fails because the three-level candidate is no longer found. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/lemonade_diagnostics.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 84993f26a4..0410cb2fef 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -581,7 +581,8 @@ mod tests { } assert_eq!( backend_candidates(root.path()), - vec![found_at_two, found_at_three] + // Sorted by path, so `a/b/...` precedes `a/build/...`. + vec![found_at_three, found_at_two] ); } From b0649452a003d32c82ade96046feeb3c71a4311f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:06:58 +0000 Subject: [PATCH 12/28] fix(e2e): fingerprint llama-server builds up to 512 MiB instead of 32 MiB A server above the old 32 MiB cap got no fingerprint, so the report would be missing its identity evidence on exactly the kind of build it is meant to describe. A dynamically linked llama-server is a few MiB, but one that links the HIP kernels for several GPU targets statically can be hundreds, and neither the repository nor this environment has the release asset sizes to settle it. The cap is now 512 MiB. Hashing already streams 64 KiB at a time and checks its deadline between reads, so memory stays bounded and time is bounded by the inventory budget; nothing about the cap changes either. The over-cap line is worded from the constant so it cannot drift from it, the boundary test uses a small limit instead of hashing half a gibibyte, and a separate test shows a file above the old cap is fingerprinted. docs/testing.md states the cap and that it is an assumption about the builds. Signed-off-by: Jussi Elo --- docs/testing.md | 18 ++-- .../e2e-cucumber/src/lemonade_diagnostics.rs | 90 +++++++++++++++---- 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3bb4c55b77..1c74bfad6d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -840,9 +840,15 @@ output, plus metadata and formatting (invalid UTF-8 bytes may expand when render Each probe has a five-second timeout. The inventory, including hashing, has a 20-second budget and the probes have their own 20-second budget, which starts when the service stop returns; hashing runs off the async executor, rejects files -over 32 MiB, and checks its deadline between 64 KiB reads. A candidate over the -size cap gets no fingerprint, and its entry says so with `sha256 unavailable: -executable exceeds 32 MiB hash limit`; it is still listed and probed. Filesystem +over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at +one buffer whatever the size. The cap sits well above a dynamically linked +`llama-server` (a few MiB, with the HIP kernels in separate libraries) and above +what a statically linked multi-target build is expected to weigh, because a +missing fingerprint on the lane this report is for would defeat it; it is an +assumption about those builds, not a measured size, and a larger file is a +finding in its own right. A candidate over the size cap gets no fingerprint, and +its entry says so with `sha256 unavailable: executable exceeds 512 MiB hash +limit`; it is still listed and probed. Filesystem operations and process cleanup are not preemptible, so this is not a hard wall-clock limit on the entire report. A candidate or configured build pin is **not proof** of which binary the router @@ -850,9 +856,9 @@ launched; a failed probe is retained as evidence, not treated as an identity. Probes inherit the harness environment rather than reconstructing the engine's managed library paths, so a probe loader error alone does not establish why the serve failed. The report reads files first and executes a backend second. The -manifest, pins, candidate paths, fingerprints and `version.txt` are read once the serve has exited -non-zero or its readiness wait has timed out and the log has been read and -archived, but before the service is stopped, so they describe the state being +manifest, pins, candidate paths, fingerprints and `version.txt` are read once the +serve has exited non-zero or its readiness wait has timed out and the log has +been read and archived, but before the service is stopped, so they describe the state being diagnosed. The `--version` and `--help` probes run after the stop returns, in the report's "probes, run after the service stop was attempted" section: llama.cpp builds commonly initialise a GPU backend even for `--version`, and doing that next diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 0410cb2fef..fe448ba910 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -29,7 +29,16 @@ const INVENTORY_BUDGET: Duration = Duration::from_secs(20); /// Time for the probes, started after the stop returns so a slow stop does not /// use it up. const PROBE_BUDGET: Duration = Duration::from_secs(20); -const MAX_HASH_BYTES: u64 = 32 * 1024 * 1024; +/// Largest executable that is fingerprinted. +/// +/// A fingerprint that is absent on the lane the report is for is a defect, so the +/// cap is set well above a dynamically linked `llama-server` (a few MiB; the HIP +/// kernels live in separate shared libraries). It is still finite: a build that +/// statically linked kernels for several GPU targets can reach hundreds of MiB, and +/// a corrupt cache entry should not be hashed at all. Hashing streams 64 KiB at a +/// time, so memory stays bounded whatever this is; the time is bounded by the +/// inventory budget, which the hashing loop checks between reads. +const MAX_HASH_BYTES: u64 = 512 * 1024 * 1024; #[derive(Clone, Copy)] struct Budgets { @@ -340,14 +349,32 @@ fn empty_search_note(runtime: &Path) -> String { ) } +/// `limit` as the report words it: whole MiB when it is one, else bytes. +fn describe_hash_limit(limit: u64) -> String { + const MIB: u64 = 1024 * 1024; + if limit.is_multiple_of(MIB) { + format!("{} MiB", limit / MIB) + } else { + format!("{limit} bytes") + } +} + fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + fingerprint_with_limit(server, deadline, MAX_HASH_BYTES) +} + +fn fingerprint_with_limit(server: &Path, deadline: Instant, limit: u64) -> std::io::Result { + let too_large = || { + std::io::Error::other(format!( + "executable exceeds {} hash limit", + describe_hash_limit(limit) + )) + }; let mut file = std::fs::File::open(server)?; - // A real server is much smaller; do not hash an arbitrarily large corrupt - // cache entry while reporting a failure. Never label a partial hash as SHA256. - if file.metadata()?.len() > MAX_HASH_BYTES { - return Err(std::io::Error::other( - "executable exceeds 32 MiB hash limit", - )); + // Do not hash an arbitrarily large corrupt cache entry while reporting a + // failure. Never label a partial hash as SHA256. + if file.metadata()?.len() > limit { + return Err(too_large()); } let mut hasher = Sha256::new(); let mut count = 0; @@ -363,10 +390,8 @@ fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { } let read = file.read(&mut buffer)?; count += read as u64; - if count > MAX_HASH_BYTES { - return Err(std::io::Error::other( - "executable exceeds 32 MiB hash limit", - )); + if count > limit { + return Err(too_large()); } if read == 0 { break; @@ -517,23 +542,51 @@ mod tests { fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { let root = tempfile::tempdir().unwrap(); let server = root.path().join("llama-server"); + let limit = 3 * 64 * 1024 + 5; let file = std::fs::File::create(&server).unwrap(); - file.set_len(MAX_HASH_BYTES).unwrap(); + file.set_len(limit).unwrap(); assert_eq!( - fingerprint(&server, Instant::now() + INVENTORY_BUDGET) + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) .unwrap() .len(), 64 ); - file.set_len(MAX_HASH_BYTES + 1).unwrap(); + file.set_len(limit + 1).unwrap(); assert_eq!( - fingerprint(&server, Instant::now() + INVENTORY_BUDGET) + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) .unwrap_err() .to_string(), - "executable exceeds 32 MiB hash limit" + format!("executable exceeds {limit} bytes hash limit") ); } + /// The cap is not the old 32 MiB: a server above it, such as a static HIP build, + /// must still get a fingerprint on the lane this report is for. The file is + /// sparse, so the test hashes zeros rather than holding real data. + #[test] + fn a_server_above_the_old_32_mib_cap_is_still_fingerprinted() { + const OLD_CAP: u64 = 32 * 1024 * 1024; + assert!(MAX_HASH_BYTES >= 256 * 1024 * 1024); + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::File::create(&server) + .unwrap() + .set_len(OLD_CAP + 1) + .unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + INVENTORY_BUDGET) + .unwrap() + .len(), + 64 + ); + } + + #[test] + fn hash_limit_is_worded_in_the_unit_it_was_set_in() { + assert_eq!(describe_hash_limit(512 * 1024 * 1024), "512 MiB"); + assert_eq!(describe_hash_limit(1000), "1000 bytes"); + } + #[test] fn hash_worker_obeys_its_own_deadline() { let root = tempfile::tempdir().unwrap(); @@ -681,7 +734,10 @@ mod tests { .unwrap(); let report = evidence(&write_manifest(root.path())).await; assert!( - report.contains("sha256 unavailable: executable exceeds 32 MiB hash limit"), + report.contains(&format!( + "sha256 unavailable: executable exceeds {} hash limit", + describe_hash_limit(MAX_HASH_BYTES) + )), "{report}" ); assert!(!report.contains("sha256: "), "{report}"); From 465912ffc861cf076fbebd5fa1672c681aa28881 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:08:34 +0000 Subject: [PATCH 13/28] test(e2e): run the probe timeout and kill path on every platform The probe's timeout, kill and reap handling was tested only through shell-script llama-server stand-ins, which exist on unix, so the Windows lane never ran it and the change did not say so. The probe is split into building the llama-server command and running a command with bounded time and output, so the second part can run any child. Two tests re-run the unit-test binary itself as that child, through an ignored helper test selected by an environment variable: one lets the probe time out and requires the child to be gone afterwards (killed and reaped, not merely abandoned), the other covers a child that exits and its status and output. Replacing the kill with a leak turns the first red. The kill/reap failed branch stays untested, because it needs an operating system that refuses to kill a child it just started, and docs/testing.md says so. Signed-off-by: Jussi Elo --- docs/testing.md | 8 ++ .../e2e-cucumber/src/lemonade_diagnostics.rs | 86 ++++++++++++++++++- 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 1c74bfad6d..2697b82e55 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -870,6 +870,14 @@ router's side of the stop. The order is fixed in one function, `backend_evidence_around_stop`, whose unit tests make the stop delete the candidate to show the fingerprint was read before it and the probes came after it. +The probe's timeout, kill and reap path is tested on every platform, Windows +included: the unit tests re-run the test binary itself as the child (an ignored +helper test selected by an environment variable), let the probe time out, and +require that the child is gone afterwards. The shell-script `llama-server` +stand-ins used by the other probe tests exist on unix only. Not covered anywhere +is the `kill/reap failed` branch, which needs an operating system that refuses to +kill a child it just started. + Use the evidence to distinguish backend startup, download, and router readiness failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the Strix Halo Ubuntu investigation. A backend's ready line does not establish that diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index fe448ba910..2d63869391 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -419,8 +419,17 @@ impl CapturedOutput { async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { let mut command = tokio::process::Command::new(server); + command.arg(flag); + run_probe(command, timeout).await +} + +/// Run `command` with bounded time and output, and render what came back. +/// +/// Split from [`probe`] so the timeout, kill and reap path can be driven with any +/// child, not only a `llama-server` stand-in: shell-script stand-ins exist on unix +/// only, and this path is what must also hold on Windows. +async fn run_probe(mut command: tokio::process::Command, timeout: Duration) -> String { command - .arg(flag) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -1056,6 +1065,81 @@ esac ); } + /// Selects what [`probe_child`] does when a probe test re-runs this test binary + /// as its child process. + const PROBE_CHILD_ENV: &str = "ROCM_E2E_PROBE_CHILD"; + + /// Not a test: the child process the probe tests below start. + /// + /// The probe's timeout, kill and reap path has to hold on Windows too, and a + /// shell-script stand-in does not exist there. This test binary does, on every + /// platform, so the tests re-run it with `--ignored --exact` and this function + /// plays the stand-in. It is `#[ignore]`d so a normal run skips it, and does + /// nothing unless [`PROBE_CHILD_ENV`] selects a behaviour. + #[test] + #[ignore = "child process of the probe tests, not a test of its own"] + fn probe_child() { + match std::env::var(PROBE_CHILD_ENV).as_deref() { + Ok("sleep") => { + println!("pid:{}", std::process::id()); + std::thread::sleep(Duration::from_secs(30)); + } + Ok("exit") => println!("child-ran"), + _ => {} + } + } + + fn probe_child_command(behaviour: &str) -> tokio::process::Command { + let module = module_path!().split_once("::").unwrap().1; + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + &format!("{module}::probe_child"), + "--ignored", + "--nocapture", + ]) + .env(PROBE_CHILD_ENV, behaviour); + command + } + + /// The timeout path on every platform: the child is still running when the + /// timeout fires, so the report must say so, return promptly, and leave no + /// process behind. `process_is_running` is false only once the child has been + /// killed and reaped, so a probe that merely abandoned it turns this red. + #[tokio::test] + async fn a_probe_that_outlives_its_timeout_is_killed_and_reaped() { + let start = Instant::now(); + let report = run_probe(probe_child_command("sleep"), Duration::from_secs(5)).await; + assert!(report.starts_with("probe timed out after 5s"), "{report}"); + assert!(!report.contains("kill/reap failed"), "{report}"); + assert!(start.elapsed() < Duration::from_secs(20), "{report}"); + let pid = report + .lines() + .find_map(|line| line.strip_prefix("pid:")) + .unwrap_or_else(|| { + panic!("the child did not report its pid before the timeout: {report}") + }) + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + /// The normal path on every platform: status and output come back, and the + /// timeout branch is not taken. + #[tokio::test] + async fn a_probe_that_exits_reports_its_status_and_output() { + let report = run_probe(probe_child_command("exit"), Duration::from_secs(60)).await; + let status = if cfg!(windows) { + "exit code: 0\n" + } else { + "exit status: 0\n" + }; + assert!(report.starts_with(status), "{report}"); + assert!(report.contains("child-ran"), "{report}"); + assert!(!report.contains("timed out"), "{report}"); + } + #[cfg(unix)] #[tokio::test] async fn probe_timeout_is_bounded() { From a9bf80789158bd7c2e575348764aafc240699d40 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:10:04 +0000 Subject: [PATCH 14/28] test(e2e): keep the revisit-before-deleting caveat in the expectations header The header rewrite for the #897 rows dropped the sentence telling the next person to revisit before deleting any of these rows, because an XPASS streak is not evidence of a fix while the lane cannot say which Lemonade build a run used. That is still true until a failed run there has carried the backend evidence this branch adds, so it is back, reworded for that. The sentence that flaky is set per row and not derived from bug was restored earlier and is unchanged. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/expectations.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 85d5642773..e6d9399780 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -68,6 +68,10 @@ reason = "A vLLM serve of a short name does not resolve to lemonade's full model # `bug`, so a new #897 row needs `flaky = true` added explicitly. Failed serves # now capture configured pins, candidate executable paths and fingerprints # before stopping the service, and each candidate's --version/--help after it. +# Revisit under EAI-7853 before deleting any of these rows: which lemonade build +# a given run used could not be read back from this lane, and until a failed run +# there has carried that evidence, an XPASS streak is not by itself evidence that +# the bug is fixed. # # os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] From 1e514a92c756943c6feb13f598c15d38b0bf35bd Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:23:18 +0000 Subject: [PATCH 15/28] fix(e2e): make the stop-ordering helper and its tests pass the CI clippy lints The CI clippy runs found what the unit tests could not: the async helper and its Probe trait were not Send, two helpers could be const, one assertion was on a constant, and a probe test spelled a minute in seconds. A Send helper cannot borrow the scenario world, which holds PTY handles and is not Sync, so the serve step now hands it the isolated root and the app paths by value, and the stop is split into one function on the world and one on the root path. The ordering tests use a mutex and an atomic instead of cells, and the size-cap assertion is a const block. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 47 +++++++++++-------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 24 ++++++++-- 2 files changed, 47 insertions(+), 24 deletions(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 2d63869391..0b16a24fa9 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -53,8 +53,13 @@ const DEFAULT_BUDGETS: Budgets = Budgets { /// Runs one `llama-server` flag and renders what came back. A trait so the order /// of probes against the service stop is testable without executing a binary. -trait Probe { - async fn run(&self, server: &Path, flag: &str, timeout: Duration) -> String; +trait Probe: Sync { + fn run( + &self, + server: &Path, + flag: &str, + timeout: Duration, + ) -> impl std::future::Future + Send; } struct RealProbe; @@ -79,9 +84,9 @@ impl Probe for RealProbe { /// The report says `engine unknown` when no engine was determined, so a skipped /// inventory stays visible, and returns no evidence for another engine. pub async fn backend_evidence_around_stop( - manifest_path: impl FnOnce() -> PathBuf, + manifest_path: impl FnOnce() -> PathBuf + Send, engine: Option<&str>, - stop: impl FnOnce() -> String, + stop: impl FnOnce() -> String + Send, ) -> (String, Option) { match engine { Some("lemonade") => { @@ -98,7 +103,7 @@ async fn evidence_around_stop( manifest_path: &Path, budgets: Budgets, probe: &impl Probe, - stop: impl FnOnce() -> String, + stop: impl FnOnce() -> String + Send, ) -> (String, String) { let inventory = take_inventory(manifest_path, budgets.inventory).await; let stop_status = stop(); @@ -114,7 +119,7 @@ struct Inventory { } impl Inventory { - fn without_probes(report: String) -> Self { + const fn without_probes(report: String) -> Self { Self { report, servers: Vec::new(), @@ -263,7 +268,7 @@ const MAX_DIR_ENTRIES: usize = 256; /// Names listed when explaining an empty search. const MAX_LISTED_NAMES: usize = 16; -fn server_binary() -> &'static str { +const fn server_binary() -> &'static str { if cfg!(windows) { "llama-server.exe" } else { @@ -575,7 +580,7 @@ mod tests { #[test] fn a_server_above_the_old_32_mib_cap_is_still_fingerprinted() { const OLD_CAP: u64 = 32 * 1024 * 1024; - assert!(MAX_HASH_BYTES >= 256 * 1024 * 1024); + const { assert!(MAX_HASH_BYTES >= 256 * 1024 * 1024) }; let root = tempfile::tempdir().unwrap(); let server = root.path().join("llama-server"); std::fs::File::create(&server) @@ -779,13 +784,15 @@ mod tests { } } + type Events = std::sync::Mutex>; + struct RecordingProbe<'a> { - events: &'a std::cell::RefCell>, + events: &'a Events, } impl Probe for RecordingProbe<'_> { async fn run(&self, _server: &Path, flag: &str, _timeout: Duration) -> String { - self.events.borrow_mut().push(format!("probe {flag}")); + self.events.lock().unwrap().push(format!("probe {flag}")); "recorded, not executed".to_owned() } } @@ -829,13 +836,13 @@ mod tests { #[tokio::test] async fn inventory_precedes_the_stop_and_probes_follow_it() { let (_root, manifest, server) = lemonade_data_dir_with_candidate(); - let events = std::cell::RefCell::new(Vec::new()); + let events = Events::default(); let (stop_status, report) = evidence_around_stop( &manifest, DEFAULT_BUDGETS, &RecordingProbe { events: &events }, || { - events.borrow_mut().push("stop".to_owned()); + events.lock().unwrap().push("stop".to_owned()); std::fs::remove_file(&server).unwrap(); "stopped".to_owned() }, @@ -843,7 +850,7 @@ mod tests { .await; assert_eq!(stop_status, "stopped"); assert_eq!( - *events.borrow(), + *events.lock().unwrap(), ["stop", "probe --version", "probe --help"], "{report}" ); @@ -860,9 +867,9 @@ mod tests { #[tokio::test] async fn stop_runs_once_for_every_engine_and_an_unknown_engine_stays_visible() { - let stops = std::cell::Cell::new(0); + let stops = std::sync::atomic::AtomicUsize::new(0); let stop = || { - stops.set(stops.get() + 1); + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); "stopped".to_owned() }; let no_manifest = || -> PathBuf { panic!("the manifest is only read for lemonade") }; @@ -878,20 +885,20 @@ mod tests { Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()) ) ); - assert_eq!(stops.get(), 2); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); let (_root, manifest, _server) = lemonade_data_dir_with_candidate(); let (status, evidence) = backend_evidence_around_stop(|| manifest, Some("lemonade"), stop).await; assert_eq!(status, "stopped"); assert!(evidence.unwrap().contains("manifest: ")); - assert_eq!(stops.get(), 3); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 3); } #[tokio::test] async fn an_unreadable_manifest_is_reported_and_the_service_is_still_stopped() { let root = tempfile::tempdir().unwrap(); - let events = std::cell::RefCell::new(Vec::new()); + let events = Events::default(); let (stop_status, report) = evidence_around_stop( &root.path().join("missing.json"), DEFAULT_BUDGETS, @@ -901,7 +908,7 @@ mod tests { .await; assert_eq!(stop_status, "stopped"); assert!(report.contains("unavailable:"), "{report}"); - assert!(events.borrow().is_empty(), "no candidate, no probe"); + assert!(events.lock().unwrap().is_empty(), "no candidate, no probe"); assert!(!report.contains("probe target:"), "{report}"); } @@ -1129,7 +1136,7 @@ esac /// timeout branch is not taken. #[tokio::test] async fn a_probe_that_exits_reports_its_status_and_output() { - let report = run_probe(probe_child_command("exit"), Duration::from_secs(60)).await; + let report = run_probe(probe_child_command("exit"), Duration::from_mins(1)).await; let status = if cfg!(windows) { "exit code: 0\n" } else { diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index b1d3f83929..3876c1d975 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -162,11 +162,20 @@ async fn serve_failure_evidence( ); // The stop sits inside this call because its position relative to the // backend probes is the point: see `backend_evidence_around_stop`. + let app_paths = world.isolated_root.is_some().then(|| world.app_paths()); + let isolated_root = world + .isolated_root + .as_ref() + .map(|root| root.path().to_owned()); let (stop_status, backend_evidence) = e2e_cucumber::lemonade_diagnostics::backend_evidence_around_stop( - || world.app_paths().engine_runtime_manifest_path("lemonade"), + move || { + app_paths + .expect("a Lemonade serve runs in an isolated scenario root") + .engine_runtime_manifest_path("lemonade") + }, engine, - || stop_scenario_services(world), + move || stop_services_in(isolated_root.as_deref()), ) .await; serve_attempt_report(&ServeAttempt { @@ -516,9 +525,16 @@ fn default_engine_serve_target() -> &'static str { /// attempt shares the device with this one, and that must be visible in the /// failure rather than inferred from a serve that looks broken. fn stop_scenario_services(world: &E2eWorld) -> String { - world.isolated_root.as_ref().map_or_else( + stop_services_in(world.isolated_root.as_ref().map(tempfile::TempDir::path)) +} + +/// [`stop_scenario_services`] on the scenario's isolated root rather than the +/// world, so a caller that must hand the stop to a `Send` closure (the world holds +/// PTY handles and is not `Sync`) can capture the path alone. +fn stop_services_in(isolated_root: Option<&std::path::Path>) -> String { + isolated_root.map_or_else( || "not attempted: scenario has no isolated root".to_owned(), - |root| crate::stop_managed_services(root.path()), + crate::stop_managed_services, ) } From ee08e62fc378ced65023b36e358ecc0d90e1472e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:24:44 +0000 Subject: [PATCH 16/28] docs(testing): record that a current ROCm llama-server fits under the old hash cap The Strix Halo Ubuntu lanes on this branch fingerprinted the rocm-stable b12109 llama-server under the former 32 MiB cap, so the cap change is margin for larger static builds and not a repair for current ones. The cap's note said the size was an assumption; it now says what was observed and what remains an assumption. Signed-off-by: Jussi Elo --- docs/testing.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index c5cf3a3b2d..f8fa77f0f8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -977,11 +977,11 @@ Each probe has a five-second timeout. The inventory, including hashing, has a when the service stop returns; hashing runs off the async executor, rejects files over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at one buffer whatever the size. The cap sits well above a dynamically linked -`llama-server` (a few MiB, with the HIP kernels in separate libraries) and above -what a statically linked multi-target build is expected to weigh, because a -missing fingerprint on the lane this report is for would defeat it; it is an -assumption about those builds, not a measured size, and a larger file is a -finding in its own right. A candidate over the size cap gets no fingerprint, and +`llama-server`: the `rocm-stable` build b12109 on the Strix Halo Ubuntu lanes +was fingerprinted under the former 32 MiB cap, so current builds fit with room to +spare. The larger cap is margin for a statically linked multi-target build, +which is an assumption about future builds, not a measured size, and a larger +file is a finding in its own right. A candidate over the size cap gets no fingerprint, and its entry says so with `sha256 unavailable: executable exceeds 512 MiB hash limit`; it is still listed and probed. Filesystem operations and process cleanup are not preemptible, so this is not a hard From 9b07eca7bdf88e7bd11bf768c77499aa0d4a953e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 16:02:31 +0000 Subject: [PATCH 17/28] fix(e2e): bound the candidate search by time and size, and keep it off the async executor The three-level candidate search added earlier ran before the inventory's deadline existed, applied the eight-candidate cap only after walking everything, read directories on an async worker, followed symlinks, and listed an unbounded bin/llamacpp when it found nothing. A large or linked cache could therefore delay the stop and the failure report well past the intended budget. The deadline now starts at inventory entry and is read at every directory and entry during the walk, through an injected clock so a test drives it without sleeping. Directory and entry limits (128 and 2048) apply to the whole search, not per directory, and the walk ends as soon as it has eight candidates, in sorted order. Symlinks are never entered or taken as a candidate, so a cycle ends and a link out of the tree is not read; the report counts them. A search cut short by the clock or a limit says so instead of claiming an empty result, and the listing of what bin/llamacpp holds reads at most 64 entries, shows 16 and says how many it left out. The manifest and pin reads, the search and each candidate's fingerprint run on blocking threads with the in-traversal deadline, since a timeout around a blocking task only drops its handle. docs/testing.md states the limits. Signed-off-by: Jussi Elo --- docs/testing.md | 25 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 592 ++++++++++++++++-- 2 files changed, 545 insertions(+), 72 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f8fa77f0f8..4a3eb8e6c7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -964,16 +964,31 @@ and `version.txt`, and each candidate's `--version` and `--help` output. Candidates are searched under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of `bin/llamacpp`, down to three directory levels below each; the cpu backend is not -searched. When nothing is found, the report lists those locations and what -`bin/llamacpp` holds, and does not guess at the cause: an interrupted download, a -layout outside the search and a build that was never installed look the same. +searched. The search is bounded as a whole, not per directory: it reads at most +128 directories and 2048 directory entries across all four backend directories, +stops as soon as it has eight candidates, and stops when the inventory's time is +up, checking the clock at every directory and entry. Siblings are visited in +sorted order, so a search that finishes inside its limits is deterministic. +Symlinks are never followed: only real directories are entered and only a regular +file named `llama-server` counts, so a link cycle ends and a link out of the +backend directory is not read; the report says how many symlinked entries it +declined to follow. When a limit or the clock ends the search early the report +says which, and does not claim an empty result. When nothing is found, the report +lists the searched locations and what `bin/llamacpp` holds (the first 16 names +out of at most 64 entries read, with a count of those omitted), and does not +guess at the cause: an interrupted download, a layout outside the search and a +build that was never installed look the same. The manifest and pin reads, the +search and each candidate's fingerprint run on blocking threads, never on an +async worker, and the search carries its own deadline because a timeout around a +blocking task would only drop its handle. At most eight candidates are hashed and probed, with two read-only probes per candidate. Each probe retains up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). -Each probe has a five-second timeout. The inventory, including hashing, has a -20-second budget and the probes have their own 20-second budget, which starts +Each probe has a five-second timeout. The inventory has a 20-second budget that +starts when it is entered and covers the manifest and pin reads, the search and +the hashing; the probes have their own 20-second budget, which starts when the service stop returns; hashing runs off the async executor, rejects files over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at one buffer whatever the size. The cap sits well above a dynamically linked diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 0b16a24fa9..f2bffd9015 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -157,22 +157,57 @@ impl Inventory { } } +/// Answers "has the inventory's time run out?". Injected so a test can drive the +/// traversal's deadline without sleeping; the real one compares against an +/// [`Instant`]. `Arc` because the scan runs on a blocking thread. +type Expired = std::sync::Arc bool + Send + Sync>; + /// Record the installed version and pins separately from executable build output: /// a pin describes what should be installed, not what a runner actually cached. /// /// Every candidate is labelled as such; finding a binary does not prove that the /// router launched it. Nothing here executes anything. No install, download, /// model load, or cache repair is done. +/// +/// The budget starts here, at entry, and covers the manifest and pin reads, the +/// directory search and the hashing alike. async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { + let deadline = Instant::now() + budget; + take_inventory_with( + manifest_path, + deadline, + SEARCH_LIMITS, + std::sync::Arc::new(move || Instant::now() >= deadline), + ) + .await +} + +/// What [`scan_runtime`] read from disk. +enum Scan { + /// The manifest could not be used; the report says why. + Unavailable(String), + Found { + /// Manifest fields and pins. + report: String, + discovery: Discovery, + /// Where an empty search looked, when it found nothing and was not cut short + /// by the clock. + empty_note: Option, + }, +} + +/// Everything the inventory reads before it hashes anything: the manifest, the +/// pins and the directory search. Blocking filesystem work, so it runs on a +/// blocking thread; it carries its own clock because a `timeout` around +/// `spawn_blocking` would only drop the join handle and leave this running. +fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() -> bool) -> Scan { let mut report = String::new(); let _ = writeln!(report, "manifest: {}", manifest_path.display()); let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { serde_json::from_slice::(&bytes).map_err(std::io::Error::other) }) { Ok(manifest) => manifest, - Err(error) => { - return Inventory::without_probes(format!("{report}unavailable: {error}")); - } + Err(error) => return Scan::Unavailable(format!("{report}unavailable: {error}")), }; for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { let _ = writeln!( @@ -185,7 +220,7 @@ async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { .get("runtime_dir") .and_then(serde_json::Value::as_str) else { - return Inventory::without_probes(format!( + return Scan::Unavailable(format!( "{report}backend inventory unavailable: no runtime_dir" )); }; @@ -200,13 +235,103 @@ async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { let _ = writeln!(report, "{}: {error}", pins.display()); } } - let candidates = backend_candidates(runtime); - if candidates.is_empty() { - let _ = writeln!(report, "{}", empty_search_note(runtime)); + let discovery = discover_candidates(runtime, limits, expired); + let empty_note = (discovery.candidates.is_empty() + && !matches!(discovery.stop, Some(Stop::Deadline))) + .then(|| empty_search_note(runtime)); + Scan::Found { + report, + discovery, + empty_note, + } +} + +/// One candidate's fingerprint and `version.txt`, rendered. Blocking reads, run on +/// a blocking thread with the inventory's deadline checked between hash reads. +fn inspect_candidate(server: &Path, deadline: Instant) -> String { + let mut report = String::new(); + match fingerprint(server, deadline) { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + } + report +} + +async fn take_inventory_with( + manifest_path: &Path, + deadline: Instant, + limits: SearchLimits, + expired: Expired, +) -> Inventory { + let scan_path = manifest_path.to_owned(); + let scan_clock = expired.clone(); + let scan = + tokio::task::spawn_blocking(move || scan_runtime(&scan_path, limits, scan_clock.as_ref())) + .await; + let (mut report, discovery, empty_note) = match scan { + Ok(Scan::Found { + report, + discovery, + empty_note, + }) => (report, discovery, empty_note), + Ok(Scan::Unavailable(report)) => return Inventory::without_probes(report), + Err(error) => { + return Inventory::without_probes(format!( + "manifest: {}\nunavailable: inventory scan failed: {error}", + manifest_path.display() + )); + } + }; + if let Some(note) = empty_note { + let _ = writeln!(report, "{note}"); + } + match discovery.stop { + Some(Stop::Deadline) => { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } + Some(Stop::Directories) => { + let _ = writeln!( + report, + "candidate search stopped: {} directories visited (limit)", + discovery.dirs_visited + ); + } + Some(Stop::Entries) => { + let _ = writeln!( + report, + "candidate search stopped: {} directory entries read (limit)", + discovery.entries_read + ); + } + Some(Stop::Candidates) | None => {} + } + if discovery.symlinks_skipped > 0 { + let _ = writeln!( + report, + "{} symlinked entries under the backend directories were not followed", + discovery.symlinks_skipped + ); } - let deadline = Instant::now() + budget; let mut servers = Vec::new(); - for server in candidates.iter().take(MAX_CANDIDATES) { + for server in &discovery.candidates { if Instant::now() >= deadline { let _ = writeln!( report, @@ -221,34 +346,18 @@ async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { ); servers.push(server.clone()); let path = server.clone(); - match tokio::task::spawn_blocking(move || fingerprint(&path, deadline)) - .await - .map_err(std::io::Error::other) - .and_then(|result| result) - { - Ok(hash) => { - let _ = writeln!(report, "sha256: {hash}"); - } + match tokio::task::spawn_blocking(move || inspect_candidate(&path, deadline)).await { + Ok(lines) => report.push_str(&lines), Err(error) => { let _ = writeln!(report, "sha256 unavailable: {error}"); } } - if let Some(parent) = server.parent() { - let version = parent.join("version.txt"); - if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { - let _ = writeln!( - report, - "{}: {}{note}", - version.display(), - String::from_utf8_lossy(&bytes) - ); - } - } } - if candidates.len() > MAX_CANDIDATES { + if matches!(discovery.stop, Some(Stop::Candidates)) { let _ = writeln!( report, - "backend inventory truncated after {MAX_CANDIDATES} candidates" + "backend inventory truncated after {MAX_CANDIDATES} candidates \ + (the search stops there; more may exist)" ); } Inventory { report, servers } @@ -262,11 +371,54 @@ const BACKEND_DIRS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan" /// unpacks some llama.cpp builds into a build-numbered directory, and the layout /// is the router's to choose, so the search is not pinned to one level. const MAX_SEARCH_DEPTH: usize = 3; -/// Directory entries read per directory, so a corrupt tree cannot make the -/// search unbounded. -const MAX_DIR_ENTRIES: usize = 256; /// Names listed when explaining an empty search. const MAX_LISTED_NAMES: usize = 16; +/// Directory entries read to build that listing. Bounded so a huge `bin/llamacpp` +/// cannot make the explanation of an empty search expensive; the report says how +/// many names it left out. +const MAX_LISTING_SCAN: usize = 64; + +/// Global bounds on one candidate search. They apply to the whole search, across +/// every backend directory, not per directory: a tree of many shallow directories +/// is as expensive as a deep one. +#[derive(Clone, Copy)] +struct SearchLimits { + /// Directories whose entries are read. + dirs: usize, + /// Directory entries read, in total. + entries: usize, + /// Candidates collected; the search stops as soon as it has this many. + candidates: usize, +} + +const SEARCH_LIMITS: SearchLimits = SearchLimits { + dirs: 128, + entries: 2048, + candidates: MAX_CANDIDATES, +}; + +/// Why a candidate search ended before it had looked everywhere. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stop { + Deadline, + Directories, + Entries, + Candidates, +} + +#[derive(Debug, Default)] +struct Discovery { + /// In a fixed order: backend directories as listed, then sibling directories + /// sorted by path. + candidates: Vec, + stop: Option, + /// Symlinks met and not followed. A cache that links out of its backend + /// directory, or back into it, is neither searched nor allowed to loop the + /// search. + symlinks_skipped: usize, + dirs_visited: usize, + entries_read: usize, +} const fn server_binary() -> &'static str { if cfg!(windows) { @@ -276,38 +428,98 @@ const fn server_binary() -> &'static str { } } -fn backend_candidates(runtime: &Path) -> Vec { - let mut candidates = Vec::new(); +/// Look for `llama-server` under each backend directory, within `limits` and until +/// `expired` says time is up. Symlinks are never followed: only real directories +/// are entered and only a regular file counts as a candidate, so a link cycle +/// ends and a link out of the tree is not read. +fn discover_candidates( + runtime: &Path, + limits: SearchLimits, + expired: &dyn Fn() -> bool, +) -> Discovery { + let mut search = Search { + limits, + expired, + out: Discovery::default(), + }; for backend in BACKEND_DIRS { - collect_candidates( - &runtime.join("bin/llamacpp").join(backend), - 0, - &mut candidates, - ); + let dir = runtime.join("bin/llamacpp").join(backend); + match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.is_dir() => search.visit(&dir, 0), + Ok(meta) if meta.file_type().is_symlink() => search.out.symlinks_skipped += 1, + _ => {} + } + if search.out.stop.is_some() { + break; + } } - candidates + search.out } -fn collect_candidates(dir: &Path, depth: usize, candidates: &mut Vec) { - let direct = dir.join(server_binary()); - if direct.is_file() { - candidates.push(direct); - } - if depth >= MAX_SEARCH_DEPTH { - return; +struct Search<'a> { + limits: SearchLimits, + expired: &'a dyn Fn() -> bool, + out: Discovery, +} + +impl Search<'_> { + fn stop(&mut self, why: Stop) { + self.out.stop.get_or_insert(why); } - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - let mut subdirs: Vec = entries - .filter_map(Result::ok) - .take(MAX_DIR_ENTRIES) - .map(|entry| entry.path()) - .filter(|path| path.is_dir()) - .collect(); - subdirs.sort(); - for subdir in subdirs { - collect_candidates(&subdir, depth + 1, candidates); + + fn visit(&mut self, dir: &Path, depth: usize) { + if self.out.stop.is_some() { + return; + } + if (self.expired)() { + self.stop(Stop::Deadline); + return; + } + if self.out.dirs_visited >= self.limits.dirs { + self.stop(Stop::Directories); + return; + } + self.out.dirs_visited += 1; + let direct = dir.join(server_binary()); + // A symlink named llama-server is not a candidate (and is counted below, + // as a directory entry, not here). + if std::fs::symlink_metadata(&direct).is_ok_and(|meta| meta.is_file()) { + self.out.candidates.push(direct); + } + if self.out.candidates.len() >= self.limits.candidates { + self.stop(Stop::Candidates); + return; + } + if depth >= MAX_SEARCH_DEPTH { + return; + } + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut subdirs = Vec::new(); + for entry in entries { + if (self.expired)() { + self.stop(Stop::Deadline); + break; + } + if self.out.entries_read >= self.limits.entries { + self.stop(Stop::Entries); + break; + } + self.out.entries_read += 1; + let Ok(entry) = entry else { + continue; + }; + match entry.file_type() { + Ok(kind) if kind.is_dir() => subdirs.push(entry.path()), + Ok(kind) if kind.is_symlink() => self.out.symlinks_skipped += 1, + _ => {} + } + } + subdirs.sort(); + for subdir in subdirs { + self.visit(&subdir, depth + 1); + } } } @@ -325,19 +537,24 @@ fn empty_search_note(runtime: &Path) -> String { let present = match std::fs::read_dir(&llamacpp) { Ok(entries) => { let mut names: Vec = entries + .take(MAX_LISTING_SCAN) .filter_map(Result::ok) .map(|entry| entry.file_name().to_string_lossy().into_owned()) .collect(); + let hit_scan_cap = names.len() >= MAX_LISTING_SCAN; names.sort(); if names.is_empty() { format!("{} is empty", llamacpp.display()) } else { - let more = names.len().saturating_sub(MAX_LISTED_NAMES); + let omitted = names.len().saturating_sub(MAX_LISTED_NAMES); names.truncate(MAX_LISTED_NAMES); - let more = if more > 0 { - format!(" and {more} more") - } else { - String::new() + let more = match (omitted, hit_scan_cap) { + (0, false) => String::new(), + (_, false) => format!(" and {omitted} more"), + (_, true) => format!( + " and at least {omitted} more (only the first {MAX_LISTING_SCAN} \ + entries were read)" + ), }; format!( "{} contains: {}{more}", @@ -505,6 +722,17 @@ async fn drain_output( mod tests { use super::*; + fn candidates(runtime: &Path) -> Vec { + discover_candidates(runtime, SEARCH_LIMITS, &|| false).candidates + } + + fn touch_server(dir: &Path) -> PathBuf { + let server = dir.join(server_binary()); + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(&server, "").unwrap(); + server + } + async fn evidence(manifest: &Path) -> String { evidence_with(manifest, DEFAULT_BUDGETS).await } @@ -630,7 +858,7 @@ mod tests { let cpu = root.path().join("bin/llamacpp/cpu").join(binary); std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); std::fs::write(cpu, "").unwrap(); - assert_eq!(backend_candidates(root.path()), wanted); + assert_eq!(candidates(root.path()), wanted); } #[test] @@ -647,7 +875,7 @@ mod tests { std::fs::write(server, "").unwrap(); } assert_eq!( - backend_candidates(root.path()), + candidates(root.path()), // Sorted by path, so `a/b/...` precedes `a/build/...`. vec![found_at_three, found_at_two] ); @@ -689,6 +917,236 @@ mod tests { assert!(report.contains("cannot be read"), "{report}"); } + fn limits(dirs: usize, entries: usize) -> SearchLimits { + SearchLimits { + dirs, + entries, + candidates: MAX_CANDIDATES, + } + } + + /// The clock is read while the tree is walked, not once before it: it reports + /// time up from its third reading on, so the walk must end partway through, + /// with what it had already found and the reason it stopped. + #[test] + fn the_clock_is_checked_during_the_walk() { + let root = tempfile::tempdir().unwrap(); + let wanted: Vec = BACKEND_DIRS + .iter() + .map(|backend| touch_server(&root.path().join("bin/llamacpp").join(backend))) + .collect(); + let calls = std::sync::atomic::AtomicUsize::new(0); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }); + assert_eq!(found.stop, Some(Stop::Deadline)); + assert_eq!(found.candidates, wanted[..1], "{found:?}"); + let all = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(all.candidates, wanted); + assert_eq!(all.stop, None); + } + + /// The same, through the inventory: a clock that is already up stops the search + /// even though the hashing deadline is far off, so the walk consults the + /// inventory's clock and not only the hash loop's. + #[tokio::test] + async fn an_inventory_whose_time_is_up_skips_the_search() { + let root = tempfile::tempdir().unwrap(); + touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| true), + ) + .await; + let report = &inventory.report; + assert!( + report.contains("remaining evidence skipped: evidence budget exhausted"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + assert!( + !report.contains("no llama-server candidates found"), + "an unfinished search must not claim to have found nothing: {report}" + ); + assert!(inventory.servers.is_empty()); + } + + /// The scan is blocking filesystem work and must not run on the async executor. + /// On a current-thread runtime the test's own thread is the executor, so a clock + /// read from any other thread shows the walk ran on a blocking one. + #[tokio::test] + async fn the_directory_scan_runs_off_the_async_executor() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("bin/llamacpp/rocm-stable/build")).unwrap(); + let threads = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let recorded = threads.clone(); + take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(move || { + recorded.lock().unwrap().push(std::thread::current().id()); + false + }), + ) + .await; + let threads = threads.lock().unwrap(); + assert!(!threads.is_empty(), "the walk never read the clock"); + let executor = std::thread::current().id(); + assert!( + threads.iter().all(|thread| *thread != executor), + "the scan ran on the async executor" + ); + } + + /// The limits cover the whole search: two backend directories with three + /// subdirectories each read six entries between them, so a limit of four stops + /// the second directory, which a per-directory limit of four would not. + #[test] + fn entry_and_directory_limits_apply_to_the_whole_search() { + let root = tempfile::tempdir().unwrap(); + for backend in ["rocm-stable", "rocm-nightly"] { + for build in ["a", "b", "c"] { + std::fs::create_dir_all(root.path().join("bin/llamacpp").join(backend).join(build)) + .unwrap(); + } + } + let found = discover_candidates(root.path(), limits(100, 4), &|| false); + assert_eq!(found.stop, Some(Stop::Entries), "{found:?}"); + assert_eq!(found.entries_read, 4, "{found:?}"); + + let found = discover_candidates(root.path(), limits(2, 100), &|| false); + assert_eq!(found.stop, Some(Stop::Directories), "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!( + found.stop, None, + "the default limits are not the small ones" + ); + } + + #[tokio::test] + async fn a_search_cut_short_by_a_limit_says_so_and_does_not_claim_an_empty_result() { + let root = tempfile::tempdir().unwrap(); + for build in ["a", "b", "c"] { + std::fs::create_dir_all(root.path().join("bin/llamacpp/rocm-stable").join(build)) + .unwrap(); + } + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + limits(2, 100), + std::sync::Arc::new(|| false), + ) + .await; + assert!( + inventory + .report + .contains("candidate search stopped: 2 directories visited (limit)"), + "{}", + inventory.report + ); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + limits(100, 2), + std::sync::Arc::new(|| false), + ) + .await; + assert!( + inventory + .report + .contains("candidate search stopped: 2 directory entries read (limit)"), + "{}", + inventory.report + ); + } + + /// The candidate cap ends the walk as soon as it is reached, instead of the walk + /// finishing and the list being cut afterwards: with twenty candidate + /// directories only the root and the first eight are entered, in sorted order. + #[test] + fn the_candidate_cap_stops_the_walk_when_it_is_reached() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + let wanted: Vec = (0..20) + .map(|i| touch_server(&backend.join(format!("{i:02}")))) + .collect(); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, Some(Stop::Candidates)); + assert_eq!(found.candidates, wanted[..MAX_CANDIDATES]); + assert_eq!(found.dirs_visited, 1 + MAX_CANDIDATES, "{found:?}"); + } + + /// A symlink is neither entered nor read as a candidate. The tree holds a link + /// back to its own backend directory (a cycle), a link out to a directory with a + /// real `llama-server`, a `llama-server` that is a link to a file outside, and a + /// whole backend directory that is a link. The walk must end, find only the real + /// file, and count what it declined to follow. + #[cfg(unix)] + #[test] + fn symlinks_are_not_followed_and_a_cycle_ends() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let outside_server = touch_server(outside.path()); + let backends = root.path().join("bin/llamacpp"); + let real = touch_server(&backends.join("rocm-stable/real")); + symlink( + backends.join("rocm-stable"), + backends.join("rocm-stable/loop"), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-stable/out")).unwrap(); + symlink( + &outside_server, + backends.join("rocm-stable").join(server_binary()), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-nightly")).unwrap(); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.candidates, vec![real], "{found:?}"); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.symlinks_skipped, 4, "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + } + + #[tokio::test] + async fn the_empty_search_listing_is_bounded_and_counts_what_it_omits() { + let root = tempfile::tempdir().unwrap(); + let llamacpp = root.path().join("bin/llamacpp"); + for i in 0..(MAX_LISTED_NAMES + 4) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + assert!(report.contains(" and 4 more"), "{report}"); + assert!(!report.contains("at least"), "{report}"); + + for i in 0..(2 * MAX_LISTING_SCAN) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + let listing = report + .split("contains: ") + .nth(1) + .and_then(|rest| rest.lines().next()) + .unwrap(); + let (names, omitted) = listing.split_once(" and at least ").unwrap(); + assert_eq!(names.split(", ").count(), MAX_LISTED_NAMES, "{report}"); + assert!( + omitted.starts_with(&format!("{} more", MAX_LISTING_SCAN - MAX_LISTED_NAMES)), + "{report}" + ); + assert!( + omitted.contains(&format!("first {MAX_LISTING_SCAN} entries")), + "{report}" + ); + } + #[tokio::test] async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { let root = tempfile::tempdir().unwrap(); From cde16c3056a3fea7fcbdba8a4c8e2e9d4ef3d5c5 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 16:31:38 +0000 Subject: [PATCH 18/28] fix(e2e): do not say the search found nothing when a limit ended it early The empty-search explanation, which lists every backend directory as searched, was suppressed only when the clock ended the search. A search stopped by the directory or entry limit still produced it, so with a real build under rocm-stable/b and the cap reached before b was entered, the report said all four backend trees had been searched and held nothing. The explanation is now emitted only when the search finished, and every early stop reports its own reason instead. The regression test uses that exact tree and asserts the empty-result claim is absent for the directory limit, the entry limit and the clock, and present for a finished search that found nothing. The candidate cap needs no row: it only stops a search that already has candidates. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 88 ++++++++++++------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index f2bffd9015..0892538330 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -236,9 +236,11 @@ fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() - } } let discovery = discover_candidates(runtime, limits, expired); - let empty_note = (discovery.candidates.is_empty() - && !matches!(discovery.stop, Some(Stop::Deadline))) - .then(|| empty_search_note(runtime)); + // "Nothing found" is only a finding when the search looked everywhere; one cut + // short by the clock or a limit reports that instead (see `take_inventory_with`). + // A candidate-cap stop has candidates by definition, so it never gets here. + let empty_note = (discovery.candidates.is_empty() && discovery.stop.is_none()) + .then(|| empty_search_note(runtime)); Scan::Found { report, discovery, @@ -1028,39 +1030,61 @@ mod tests { ); } + /// A search that did not finish must not say it found nothing. The tree is the + /// one a reviewer described: an empty `a`, and a real build under `b`, with the + /// cap reached before `b` is entered. Each way a search can stop early reports + /// its own reason and never the completed-empty-search text, which is asserted + /// absent; a search that did finish and found nothing still gets it. (The + /// candidate cap has no row: it stops a search that already has candidates.) #[tokio::test] - async fn a_search_cut_short_by_a_limit_says_so_and_does_not_claim_an_empty_result() { + async fn a_search_cut_short_says_why_and_never_claims_an_empty_result() { + const EMPTY_CLAIM: &str = "no llama-server candidates found"; let root = tempfile::tempdir().unwrap(); - for build in ["a", "b", "c"] { - std::fs::create_dir_all(root.path().join("bin/llamacpp/rocm-stable").join(build)) - .unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(backend.join("a")).unwrap(); + touch_server(&backend.join("b")); + let manifest = write_manifest(root.path()); + let far = Instant::now() + INVENTORY_BUDGET; + let never: Expired = std::sync::Arc::new(|| false); + let always: Expired = std::sync::Arc::new(|| true); + + for (stop, search_limits, clock, stop_line) in [ + ( + Stop::Directories, + limits(2, 100), + never.clone(), + "candidate search stopped: 2 directories visited (limit)", + ), + ( + Stop::Entries, + limits(100, 1), + never.clone(), + "candidate search stopped: 1 directory entries read (limit)", + ), + ( + Stop::Deadline, + SEARCH_LIMITS, + always, + "remaining evidence skipped: evidence budget exhausted", + ), + ] { + let inventory = take_inventory_with(&manifest, far, search_limits, clock).await; + let report = &inventory.report; + assert!(report.contains(stop_line), "{stop:?}: {report}"); + assert!(!report.contains(EMPTY_CLAIM), "{stop:?}: {report}"); + assert!( + !report.contains("candidate (not proof"), + "{stop:?}: {report}" + ); } - let inventory = take_inventory_with( - &write_manifest(root.path()), - Instant::now() + INVENTORY_BUDGET, - limits(2, 100), - std::sync::Arc::new(|| false), - ) - .await; - assert!( - inventory - .report - .contains("candidate search stopped: 2 directories visited (limit)"), - "{}", - inventory.report - ); - let inventory = take_inventory_with( - &write_manifest(root.path()), - Instant::now() + INVENTORY_BUDGET, - limits(100, 2), - std::sync::Arc::new(|| false), - ) - .await; + + let finished = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(finished.path().join("bin/llamacpp/rocm-stable/a")).unwrap(); + let inventory = + take_inventory_with(&write_manifest(finished.path()), far, SEARCH_LIMITS, never).await; assert!( - inventory - .report - .contains("candidate search stopped: 2 directory entries read (limit)"), - "{}", + inventory.report.contains(EMPTY_CLAIM), + "a search that finished and found nothing says so: {}", inventory.report ); } From f013eae042247b9641d2d108eb90735522bfa47f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 16:34:46 +0000 Subject: [PATCH 19/28] fix(e2e): find the helper child's pid when libtest runs it on one test thread The probe timeout test reads the pid its helper child prints. With one test thread (RUST_TEST_THREADS=1, or a one-CPU host) libtest writes the test-name label first and the child's output continues that line, so the line-start match for the marker found nothing and the test failed even though the child had been killed and reaped. The child now starts a fresh line before the marker, and the test runs the child both on one test thread and on several (overlapping, so it costs one timeout), so the single-thread case is exercised wherever the suite runs. With the newline removed the single-thread run fails; it failed the same way before this change under RUST_TEST_THREADS=1. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 40 ++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 0892538330..63d2976136 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -1570,7 +1570,11 @@ esac fn probe_child() { match std::env::var(PROBE_CHILD_ENV).as_deref() { Ok("sleep") => { - println!("pid:{}", std::process::id()); + // With one test thread (RUST_TEST_THREADS=1, or one CPU) libtest + // writes "test ... " first and our output continues that + // line, so start a fresh one: the parent reads the first line that + // begins with the marker. + println!("\npid:{}", std::process::id()); std::thread::sleep(Duration::from_secs(30)); } Ok("exit") => println!("child-ran"), @@ -1592,14 +1596,16 @@ esac command } - /// The timeout path on every platform: the child is still running when the - /// timeout fires, so the report must say so, return promptly, and leave no - /// process behind. `process_is_running` is false only once the child has been - /// killed and reaped, so a probe that merely abandoned it turns this red. - #[tokio::test] - async fn a_probe_that_outlives_its_timeout_is_killed_and_reaped() { + /// One run of the timeout path against a helper child that libtest runs with + /// `test_threads` test threads: the child is still running when the timeout + /// fires, so the report must say so, return promptly, and leave no process + /// behind. `process_is_running` is false only once the child has been killed + /// and reaped, so a probe that merely abandoned it turns this red. + async fn assert_timeout_kills_and_reaps(test_threads: &str) { let start = Instant::now(); - let report = run_probe(probe_child_command("sleep"), Duration::from_secs(5)).await; + let mut command = probe_child_command("sleep"); + command.env("RUST_TEST_THREADS", test_threads); + let report = run_probe(command, Duration::from_secs(5)).await; assert!(report.starts_with("probe timed out after 5s"), "{report}"); assert!(!report.contains("kill/reap failed"), "{report}"); assert!(start.elapsed() < Duration::from_secs(20), "{report}"); @@ -1607,13 +1613,29 @@ esac .lines() .find_map(|line| line.strip_prefix("pid:")) .unwrap_or_else(|| { - panic!("the child did not report its pid before the timeout: {report}") + panic!( + "the child did not report its pid before the timeout \ + (test threads {test_threads}): {report}" + ) }) .parse::() .unwrap(); assert!(!rocm_core::process_is_running(pid), "{report}"); } + /// The timeout path on every platform, with the child run both on one test + /// thread and on several. With one thread (also what a one-CPU host gets) libtest + /// prefixes the child's output with its test-name label, so a pid printed on + /// that same line is not found by a line-start match. The two runs overlap, so + /// this takes one timeout, not two. + #[tokio::test] + async fn a_probe_that_outlives_its_timeout_is_killed_and_reaped() { + tokio::join!( + assert_timeout_kills_and_reaps("1"), + assert_timeout_kills_and_reaps("8"), + ); + } + /// The normal path on every platform: status and output come back, and the /// timeout branch is not taken. #[tokio::test] From d4ca134fed221b0b61c1381c57b183fa7ce72088 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 17:02:15 +0000 Subject: [PATCH 20/28] fix(e2e): name the engine install manifest path so the registry writer guard does not match it The registry guard in runtime_registry.rs reads each production file as text and flags one that both writes a file and spells runtime_manifest_path(. The shared helper added for the Lemonade install manifest was called engine_runtime_manifest_path, which contains that spelling, and the Lemonade installer both calls it and writes its manifest, so the guard reported a second registry writer. It only showed once the guard reached main. The helper is now AppPaths::engine_install_manifest_path, which also says what it is: the engine's own install manifest, not an entry in the runtime registry. The doc comment records why the names must stay apart. The path it returns is unchanged, and so is the guard. Signed-off-by: Jussi Elo --- crates/rocm-core/src/paths.rs | 11 ++++++++--- engines/lemonade/src/install.rs | 4 ++-- tests/e2e-cucumber/src/lemonade_diagnostics.rs | 2 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 7a2ff4d2c1..7d526244e4 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -200,7 +200,12 @@ impl AppPaths { /// file, so a reader cannot drift to a different name: a reader that /// misses it degrades to "no manifest", which is indistinguishable from an /// engine that was never installed. - pub fn engine_runtime_manifest_path(&self, engine: &str) -> PathBuf { + /// + /// This is the engine's own install manifest, not an entry in the runtime + /// registry (that is `apps/rocm`'s `runtime_manifest_path`), and the name keeps + /// the two apart: the registry has a text guard that treats any file spelling + /// that helper's name and writing a file as a second registry writer. + pub fn engine_install_manifest_path(&self, engine: &str) -> PathBuf { self.engine_manifests_dir(engine).join("runtime.json") } @@ -366,10 +371,10 @@ mod tests { ); } #[test] - fn engine_runtime_manifest_lives_in_the_engine_manifests_dir() { + fn engine_install_manifest_lives_in_the_engine_manifests_dir() { let (_root, paths) = temp_app_paths("engine-runtime-manifest-path"); assert_eq!( - paths.engine_runtime_manifest_path("lemonade"), + paths.engine_install_manifest_path("lemonade"), paths.engine_manifests_dir("lemonade").join("runtime.json") ); } diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index 2412131787..5c67cbcad4 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -196,7 +196,7 @@ where } fn manifest_path(paths: &AppPaths) -> PathBuf { - paths.engine_runtime_manifest_path(ENGINE_NAME) + paths.engine_install_manifest_path(ENGINE_NAME) } fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { @@ -597,7 +597,7 @@ mod tests { }; write_manifest(&paths, &manifest).unwrap(); - let path = paths.engine_runtime_manifest_path("lemonade"); + let path = paths.engine_install_manifest_path("lemonade"); assert_eq!(manifest_path(&paths), path); let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); assert_eq!(written["version"], "fixture"); diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 63d2976136..c355c6656d 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -1297,7 +1297,7 @@ mod tests { let server = runtime.join("bin/llamacpp/rocm-stable").join(binary); std::fs::create_dir_all(server.parent().unwrap()).unwrap(); std::fs::write(&server, b"abc").unwrap(); - let manifest = paths.engine_runtime_manifest_path("lemonade"); + let manifest = paths.engine_install_manifest_path("lemonade"); std::fs::create_dir_all(manifest.parent().unwrap()).unwrap(); std::fs::write( &manifest, diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index ec1a9e782b..2062df7641 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -172,7 +172,7 @@ async fn serve_failure_evidence( move || { app_paths .expect("a Lemonade serve runs in an isolated scenario root") - .engine_runtime_manifest_path("lemonade") + .engine_install_manifest_path("lemonade") }, engine, move || stop_services_in(isolated_root.as_deref()), From 87402100c39def4da08d91db089a4e61e407feb8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:33:11 +0000 Subject: [PATCH 21/28] fix(e2e): refuse symlinked files the report reads, not only symlinked directories The directory walk counted a symlinked version.txt as a link it did not follow, but the report then read version.txt through read_clamped, which follows it, so a real llama-server beside a marker linking outside the backend directory put the outside file's content into the report while the report said the link was skipped. Every file the report reads from the tree now goes through the same rule: the manifest, the pins file, version.txt and a candidate's fingerprint target are refused when the last path component is a symlink, and the report says "symlink not followed" in place of the content. The symlink regression fixture gains the real server with a version marker, pins file and manifest that link outside the tree; the test asserts the outside content is absent and each refusal is named. Letting the link be followed again turns it red. Signed-off-by: Jussi Elo --- docs/testing.md | 5 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 151 ++++++++++++++++-- 2 files changed, 146 insertions(+), 10 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 4dd47be3d0..a91b7ec0e4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1063,7 +1063,10 @@ sorted order, so a search that finishes inside its limits is deterministic. Symlinks are never followed: only real directories are entered and only a regular file named `llama-server` counts, so a link cycle ends and a link out of the backend directory is not read; the report says how many symlinked entries it -declined to follow. When a limit or the clock ends the search early the report +declined to follow. The same rule covers every file the report reads: the +manifest, the pins file, `version.txt` and a candidate's fingerprint target are +refused when the last path component is a symlink, and the report says `symlink +not followed` in place of the content. When a limit or the clock ends the search early the report says which, and does not claim an empty result. When nothing is found, the report lists the searched locations and what `bin/llamacpp` holds (the first 16 names out of at most 64 entries read, with a count of those omitted), and does not diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index c355c6656d..7eb8a9b945 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -157,6 +157,24 @@ impl Inventory { } } +/// What the report says about a file it did not read because it is a symlink. +const SYMLINK_NOT_FOLLOWED: &str = "symlink not followed"; + +/// Whether the last component of `path` is itself a symlink (not followed). +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +/// [`read_clamped`], except that a path which is a symlink is refused rather than +/// followed, the same rule the directory walk applies: a link inside the backend +/// tree that points outside it must not put the outside file into the report. +fn read_regular_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + if is_symlink(path) { + return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); + } + read_clamped(path, max) +} + /// Answers "has the inventory's time run out?". Injected so a test can drive the /// traversal's deadline without sleeping; the real one compares against an /// [`Instant`]. `Arc` because the scan runs on a blocking thread. @@ -203,12 +221,13 @@ enum Scan { fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() -> bool) -> Scan { let mut report = String::new(); let _ = writeln!(report, "manifest: {}", manifest_path.display()); - let manifest = match read_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { - serde_json::from_slice::(&bytes).map_err(std::io::Error::other) - }) { - Ok(manifest) => manifest, - Err(error) => return Scan::Unavailable(format!("{report}unavailable: {error}")), - }; + let manifest = + match read_regular_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return Scan::Unavailable(format!("{report}unavailable: {error}")), + }; for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { let _ = writeln!( report, @@ -227,7 +246,7 @@ fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() - let runtime = Path::new(runtime); let pins = runtime.join("resources/backend_versions.json"); let _ = writeln!(report, "configured pins (not executable identity):"); - match read_clamped(&pins, MAX_EVIDENCE_BYTES) { + match read_regular_clamped(&pins, MAX_EVIDENCE_BYTES) { Ok((bytes, note)) => { let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); } @@ -262,7 +281,9 @@ fn inspect_candidate(server: &Path, deadline: Instant) -> String { } if let Some(parent) = server.parent() { let version = parent.join("version.txt"); - if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { + if is_symlink(&version) { + let _ = writeln!(report, "{}: {SYMLINK_NOT_FOLLOWED}", version.display()); + } else if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { let _ = writeln!( report, "{}: {}{note}", @@ -594,6 +615,9 @@ fn fingerprint_with_limit(server: &Path, deadline: Instant, limit: u64) -> std:: describe_hash_limit(limit) )) }; + if is_symlink(server) { + return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); + } let mut file = std::fs::File::open(server)?; // Do not hash an arbitrarily large corrupt cache entry while reporting a // failure. Never label a partial hash as SHA256. @@ -1131,14 +1155,123 @@ mod tests { ) .unwrap(); symlink(outside.path(), backends.join("rocm-nightly")).unwrap(); + // A version marker beside the real server that links out of the tree. + std::fs::write(outside.path().join("version.txt"), "OUTSIDE-VERSION").unwrap(); + symlink( + outside.path().join("version.txt"), + backends.join("rocm-stable/real/version.txt"), + ) + .unwrap(); let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); assert_eq!(found.candidates, vec![real], "{found:?}"); assert_eq!(found.stop, None, "{found:?}"); - assert_eq!(found.symlinks_skipped, 4, "{found:?}"); + assert_eq!(found.symlinks_skipped, 5, "{found:?}"); assert_eq!(found.dirs_visited, 2, "{found:?}"); } + /// Every file the report reads from the backend tree is refused when it is a + /// symlink, and the report says so, the way the walk reports a link it did not + /// enter. A real `llama-server` sits beside a `version.txt` that links to a file + /// outside the tree, the pins file is a link out too, and so is a manifest: none + /// of the outside content may appear, and each refusal is named. A fingerprint + /// of a path that has become a link is refused as well. + #[cfg(unix)] + #[tokio::test] + async fn linked_files_are_refused_and_named_not_embedded() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + for (name, text) in [ + ("version.txt", "OUTSIDE-VERSION"), + ("pins.json", "OUTSIDE-PINS"), + ("manifest.json", r#"{"version": "OUTSIDE-MANIFEST"}"#), + ("binary", "OUTSIDE-BINARY"), + ] { + std::fs::write(outside.path().join(name), text).unwrap(); + } + let server = touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + std::fs::write(&server, "abc").unwrap(); + symlink( + outside.path().join("version.txt"), + server.parent().unwrap().join("version.txt"), + ) + .unwrap(); + std::fs::create_dir_all(root.path().join("resources")).unwrap(); + symlink( + outside.path().join("pins.json"), + root.path().join("resources/backend_versions.json"), + ) + .unwrap(); + + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + let report = &inventory.report; + assert!(!report.contains("OUTSIDE"), "{report}"); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + server.parent().unwrap().join("version.txt").display() + )), + "{report}" + ); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + root.path() + .join("resources/backend_versions.json") + .display() + )), + "{report}" + ); + // The real server beside the links is still fingerprinted, and the walk + // counts the one link it saw among the entries. + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!( + report.contains("1 symlinked entries under the backend directories were not followed"), + "{report}" + ); + + let linked_manifest = root.path().join("linked-manifest.json"); + symlink(outside.path().join("manifest.json"), &linked_manifest).unwrap(); + let inventory = take_inventory_with( + &linked_manifest, + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + assert!( + !inventory.report.contains("OUTSIDE"), + "{}", + inventory.report + ); + assert!( + inventory + .report + .contains(&format!("unavailable: {SYMLINK_NOT_FOLLOWED}")), + "{}", + inventory.report + ); + + let linked_binary = root.path().join("linked-binary"); + symlink(outside.path().join("binary"), &linked_binary).unwrap(); + assert_eq!( + fingerprint(&linked_binary, Instant::now() + INVENTORY_BUDGET) + .unwrap_err() + .to_string(), + SYMLINK_NOT_FOLLOWED + ); + } + #[tokio::test] async fn the_empty_search_listing_is_bounded_and_counts_what_it_omits() { let root = tempfile::tempdir().unwrap(); From 96f67f0b517376677f71bb5944a84047de586602 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:28:43 +0000 Subject: [PATCH 22/28] fix(e2e): keep the stop and the failure report when the Lemonade manifest lookup fails The serve step looked the runtime manifest up with an expect that runs before the service is stopped. For a Lemonade serve without an isolated scenario root it panicked, so the stop never ran and the whole failure report was lost, where the code before the evidence change reported "not attempted: scenario has no isolated root". The lookup now returns Err(reason) when there is nothing to look for, and a panic inside it is caught and treated the same way. The report says "backend evidence unavailable: ", the stop still runs exactly once, and nothing is probed. The only other expect on that path is on a child pipe that is configured to be piped and runs after the stop. The test covers both the Err and the panic and goes red when the panic guard is removed. Signed-off-by: Jussi Elo --- docs/testing.md | 7 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 69 +++++++++++++++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 4 +- 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index a91b7ec0e4..78b66be84e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1112,7 +1112,12 @@ report's `stop` section says so, and the probes still run. No lane has shown whether a probe initialises a GPU backend, which is why they are kept off the router's side of the stop. The order is fixed in one function, `backend_evidence_around_stop`, whose unit tests make the stop delete the candidate -to show the fingerprint was read before it and the probes came after it. +to show the fingerprint was read before it and the probes came after it. Nothing +before the stop may keep it from running: a serve with no isolated scenario root +has no manifest to look for, so the report says `backend evidence unavailable: +the scenario has no isolated root` (and the stop's own status says it was not +attempted), and a panic while looking the manifest up is caught and reported the +same way, so the stop runs exactly once and the failure report is not lost. The probe's timeout, kill and reap path is tested on every platform, Windows included: the unit tests re-run the test binary itself as the child (an ignored diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 7eb8a9b945..bd3a355b8e 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -79,20 +79,35 @@ impl Probe for RealProbe { /// binary, and llama.cpp builds commonly initialise a GPU backend even for /// `--version`, so running one next to a still-live router on a shared-memory /// device could change the very state being diagnosed. Reading files cannot. -/// `manifest_path` is only called for Lemonade. +/// +/// `manifest_path` is only called for Lemonade, and nothing it does may keep the +/// stop from running: it returns `Err(reason)` when there is no manifest to look +/// for (the report then says `unavailable: `), and a panic inside it is +/// caught and reported the same way, so the stop still runs exactly once and the +/// failure report survives. The report is the one thing a failed serve has left. /// /// The report says `engine unknown` when no engine was determined, so a skipped /// inventory stays visible, and returns no evidence for another engine. pub async fn backend_evidence_around_stop( - manifest_path: impl FnOnce() -> PathBuf + Send, + manifest_path: impl FnOnce() -> Result + Send, engine: Option<&str>, stop: impl FnOnce() -> String + Send, ) -> (String, Option) { match engine { Some("lemonade") => { - let (stop_status, evidence) = - evidence_around_stop(&manifest_path(), DEFAULT_BUDGETS, &RealProbe, stop).await; - (stop_status, Some(evidence)) + let manifest = std::panic::catch_unwind(std::panic::AssertUnwindSafe(manifest_path)) + .unwrap_or_else(|_| Err("finding the runtime manifest failed".to_owned())); + match manifest { + Ok(manifest) => { + let (stop_status, evidence) = + evidence_around_stop(&manifest, DEFAULT_BUDGETS, &RealProbe, stop).await; + (stop_status, Some(evidence)) + } + Err(reason) => ( + stop(), + Some(format!("backend evidence unavailable: {reason}")), + ), + } } Some(_) => (stop(), None), None => (stop(), Some(ENGINE_UNKNOWN_EVIDENCE.to_owned())), @@ -1487,7 +1502,8 @@ mod tests { stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); "stopped".to_owned() }; - let no_manifest = || -> PathBuf { panic!("the manifest is only read for lemonade") }; + let no_manifest = + || -> Result { panic!("the manifest is only read for lemonade") }; assert_eq!( backend_evidence_around_stop(no_manifest, Some("vllm"), stop).await, @@ -1504,12 +1520,51 @@ mod tests { let (_root, manifest, _server) = lemonade_data_dir_with_candidate(); let (status, evidence) = - backend_evidence_around_stop(|| manifest, Some("lemonade"), stop).await; + backend_evidence_around_stop(|| Ok(manifest), Some("lemonade"), stop).await; assert_eq!(status, "stopped"); assert!(evidence.unwrap().contains("manifest: ")); assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 3); } + /// A Lemonade serve with nothing to look the manifest up in (the scenario has no + /// isolated root) used to panic before the stop, so the stop never ran and the + /// whole failure report was lost. The lookup now fails soft, whether it says so + /// (`Err`) or panics, and the stop still runs exactly once. Removing the panic + /// guard turns the second case red. + #[tokio::test] + async fn a_failing_manifest_lookup_still_runs_the_stop_once_and_reports_why() { + let stops = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "stopped".to_owned() + }; + let (status, evidence) = backend_evidence_around_stop( + || Err("the scenario has no isolated root".to_owned()), + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: the scenario has no isolated root") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 1); + + let (status, evidence) = backend_evidence_around_stop( + || -> Result { panic!("no isolated root") }, + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: finding the runtime manifest failed") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); + } + #[tokio::test] async fn an_unreadable_manifest_is_reported_and_the_service_is_still_stopped() { let root = tempfile::tempdir().unwrap(); diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 2062df7641..d3e236b973 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -171,8 +171,8 @@ async fn serve_failure_evidence( e2e_cucumber::lemonade_diagnostics::backend_evidence_around_stop( move || { app_paths - .expect("a Lemonade serve runs in an isolated scenario root") - .engine_install_manifest_path("lemonade") + .map(|paths| paths.engine_install_manifest_path("lemonade")) + .ok_or_else(|| "the scenario has no isolated root".to_owned()) }, engine, move || stop_services_in(isolated_root.as_deref()), From 2c5a106f010e4215668c3bf0c01187165bebb725 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:33:31 +0000 Subject: [PATCH 23/28] fix(e2e): say so when a log grew past the cap after its size was read read_clamped read a file whose size fit the cap with take(max), so a log that grew after the size was taken was cut at the cap with no note, for the archived service log and for the pins and version.txt in the Lemonade report alike. The note existed only on the branch for a file that already exceeded the cap. The read now asks for one byte past the cap. If it is there, the file grew, and it is clamped the way an oversized file is, head and tail with the gap marked, and the note says the file grew while it was read. A file that fits is still returned verbatim with no note. The size is a parameter of an inner function so a test can hand it a stale one; with the extra byte not requested the test is red. All callers consume the note (the manifest read discards it, but a cut JSON manifest fails to parse and the report says so). Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/serve_log.rs | 91 ++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index a9734d1cd9..e97d62164d 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -157,16 +157,55 @@ fn slugify(name: &str) -> String { /// this is collecting. Seeking keeps the cost fixed however big the log grows. /// /// The log may still be being written, so `len` is a snapshot: the head and tail -/// are always real, and only the elided count can be slightly stale. +/// are always real, and only the elided count can be slightly stale. A log that +/// grows past the cap after the size was taken is clamped the same way and the +/// note says it grew (see [`read_clamped_file`]); no caller gets a silent +/// truncation. Callers: `archive_service_log` (the note lands on the archived-path +/// line) and the Lemonade report's pins and `version.txt` reads (the note follows +/// the quoted text); its manifest read discards the note, but a truncated JSON +/// manifest fails to parse and the report says so. pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = std::fs::File::open(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) +} + +/// [`read_clamped`] on an open file whose size was read as `len`. Split out so a +/// test can hand it a `len` that has gone stale, which is what a log that grows +/// between the size check and the read looks like. +/// +/// A file that fits is read in full, but one byte past `max` is asked for: if it +/// is there, the log grew after `len` was taken. That is not allowed to pass as +/// "fits" with the surplus dropped without a word, so the read falls back to the +/// head and tail of the file as it is now and the note says it grew. A file that +/// shrank instead is returned as read, with a note that it changed. +fn read_clamped_file( + mut file: std::fs::File, + len: u64, + max: u64, +) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; - let mut file = std::fs::File::open(path)?; - let len = file.metadata()?.len(); + let mut len = len; + let mut grown = false; if len <= max { let mut body = Vec::new(); - file.take(max).read_to_end(&mut body)?; - return Ok((body, String::new())); + file.by_ref() + .take(max.saturating_add(1)) + .read_to_end(&mut body)?; + if body.len() as u64 <= max { + return Ok((body, String::new())); + } + len = file.metadata()?.len(); + if len <= max { + body.truncate(usize::try_from(max).unwrap_or(usize::MAX)); + return Ok(( + body, + format!(" (truncated at {max} bytes: the file changed while it was read)"), + )); + } + file.seek(SeekFrom::Start(0))?; + grown = true; } // `len > max >= 2 * half`, so the two halves can never overlap. let half = max / 2; @@ -181,7 +220,15 @@ pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, S let mut body = head; body.extend_from_slice(marker.as_bytes()); body.extend_from_slice(&tail); - Ok((body, format!(" ({elided} bytes elided from the middle)"))) + let grown_note = if grown { + "; the file grew while it was read" + } else { + "" + }; + Ok(( + body, + format!(" ({elided} bytes elided from the middle{grown_note})"), + )) } /// Copy the log written by the managed service `serve_stdout` launched into @@ -302,8 +349,8 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { mod tests { use super::{ MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, - parse_log_path, read_clamped, reported_engine, serve_attempt_report, service_log_tail, - slugify, tail_lines, + parse_log_path, read_clamped, read_clamped_file, reported_engine, serve_attempt_report, + service_log_tail, slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -473,6 +520,34 @@ managed service launched read_clamped(&path, max).expect("read log") } + /// A log that grows after its size was read must not be cut at the cap without + /// saying so. `len` is handed in stale (10 bytes under a 20-byte cap) for a file + /// that now holds 60: the read keeps the head and tail of what is there and the + /// note says it grew. Cutting at the cap silently turns this red, and a file + /// that really fits (stale or not) stays verbatim with no note. + #[test] + fn a_log_that_grew_after_its_size_was_read_is_clamped_and_says_so() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("service.log"); + let body: Vec = (0..60u8).map(|i| b'a' + i % 26).collect(); + std::fs::write(&path, &body).expect("write log"); + + let (clamped, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 10, 20).expect("read"); + assert!(note.contains("the file grew while it was read"), "{note}"); + assert!(note.contains("40 bytes elided from the middle"), "{note}"); + let text = String::from_utf8_lossy(&clamped); + assert!(text.starts_with(&String::from_utf8_lossy(&body[..10]).to_string())); + assert!(text.ends_with(&String::from_utf8_lossy(&body[50..]).to_string())); + + let (whole, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 100).expect("read"); + assert_eq!((whole, note), (body.clone(), String::new())); + let (exact, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 60).expect("read"); + assert_eq!((exact, note), (body, String::new())); + } + /// A runaway log is bounded, but never at the cost of the startup banner: /// both ends survive and the gap says so. #[test] From 2840f96f1f03341ddad1a95482c7b77f6c054059 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:36:37 +0000 Subject: [PATCH 24/28] test(e2e): point the Lemonade serve xfail rows at the open ticket GitHub issue #897 is closed, but the xfail rows, their header comments, the harness tests that pin them and the docs still named it as the live tracker. The open ticket is ROCMAI-898, so every row, comment, test and doc line that named #897 as the tracker now names ROCMAI-898, with one note in the header that it was first tracked as #897 and that issue is closed. The issue stays closed. The stale-xfail hint reads the ticket ids out of the matrix and hard-codes none, so it follows the new id on its own; its self-test now covers that (a PR that mentions ROCMAI-898 is told about the rows, one that mentions #897 is not). The rows' conditions, timeouts and flaky handling are unchanged. Signed-off-by: Jussi Elo --- docs/testing.md | 10 ++--- scripts/xfail_expectations_hint.py | 14 +++++++ tests/e2e-cucumber/expectations.toml | 37 +++++++++++-------- .../features/model_serving.feature | 2 +- tests/e2e-cucumber/src/expectation.rs | 12 +++--- 5 files changed, 47 insertions(+), 28 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 78b66be84e..360e5d54a1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1128,10 +1128,10 @@ is the `kill/reap failed` branch, which needs an operating system that refuses t kill a child it just started. Use the evidence to distinguish backend startup, download, and router readiness -failures; [issue #897](https://github.com/ROCm/rocm-cli/issues/897) tracks the -Strix Halo Ubuntu investigation. A backend's ready line does not establish that -the router serves the requested model, and a readiness timeout alone does not -establish a shutdown. Diagnostics do not change the CLI's asynchronous +failures; ROCMAI-898 tracks the Strix Halo Ubuntu investigation (first tracked +as [issue #897](https://github.com/ROCm/rocm-cli/issues/897), which is closed). A +backend's ready line does not establish that the router serves the requested +model, and a readiness timeout alone does not establish a shutdown. Diagnostics do not change the CLI's asynchronous managed-launch contract or imply a backend fix. Exercise the diagnostics and expectation scoping without a GPU: @@ -2128,7 +2128,7 @@ exit) and `serve-40` in (GPU lane: serving with no watcher enabled starts no helper; on a Linux host that runs it with lemonade it is an expected failure in [expectations.toml](../tests/e2e-cucumber/expectations.toml) while lemonade -serving on the Strix Halo Ubuntu lanes is broken (#897), so the helper check is +serving on the Strix Halo Ubuntu lanes is broken (ROCMAI-898), so the helper check is not reached on those lanes. The Strix Halo Windows lane runs it and passes, and the row also matches other Linux lemonade hosts, where a pass is tolerated). diff --git a/scripts/xfail_expectations_hint.py b/scripts/xfail_expectations_hint.py index 7a8439f6ce..ff38838d13 100644 --- a/scripts/xfail_expectations_hint.py +++ b/scripts/xfail_expectations_hint.py @@ -143,6 +143,11 @@ def emit(hits: dict[str, list[str]]) -> None: when = {} bug = "#123" reason = "example" + +[["serve-lemonade-inference"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "example" """ @@ -188,6 +193,15 @@ def check(title="", body="", commits=""): assert hits == {"EAI-7333": ["serve-vllm-inference"]}, hits print("self-test: refs read from commit messages ok") + # A row keyed to an open tracker id follows that id, and an id that is no + # longer in any row (a closed issue the rows were repointed away from) does + # not match: the hint reads the ids out of the matrix, it hard-codes none. + hits = check(body="Refs ROCMAI-898") + assert hits == {"ROCMAI-898": ["serve-lemonade-inference"]}, hits + hits = check(body="Refs #897") + assert hits == {}, hits + print("self-test: hint follows the id in the matrix (ROCMAI-898 yes, #897 no) ok") + print("xfail hint self-test: ok") diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index e6d9399780..fcb0b52a4e 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -58,16 +58,19 @@ when = {} bug = "EAI-7219" reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." -# ROCm/rocm-cli#897 tracks multiple Strix Halo Ubuntu Lemonade serve signatures: +# ROCMAI-898 tracks multiple Strix Halo Ubuntu Lemonade serve signatures (first +# tracked as GitHub issue ROCm/rocm-cli#897, since closed; ROCMAI-898 is the open +# ticket and what the rows below name): # the child rejects --no-mmap (41984921053), backend readiness on :8001 without # the model appearing on :11435 (41984503055), and backend download failure # (41986772728). A backend ready line does not prove a later shutdown. # EAI-7423 described only the earlier ready-then-unavailable symptom. # Retain flaky=true because historical runs also passed intermittently; do not # infer a fix from a single XPASS. The flag is per-row and is NOT derived from -# `bug`, so a new #897 row needs `flaky = true` added explicitly. Failed serves -# now capture configured pins, candidate executable paths and fingerprints -# before stopping the service, and each candidate's --version/--help after it. +# `bug`, so a new ROCMAI-898 row needs `flaky = true` added explicitly. +# Failed serves now capture configured pins, candidate executable paths and +# fingerprints before stopping the service, and each candidate's --version/--help +# after it. # Revisit under EAI-7853 before deleting any of these rows: which lemonade build # a given run used could not be read back from this lane, and until a failed run # there has carried that evidence, an XPASS streak is not by itself evidence that @@ -76,7 +79,7 @@ reason = "A vLLM serve of a short name does not resolve to lemonade's full model # os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -84,7 +87,7 @@ flaky = true # The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -92,7 +95,7 @@ flaky = true # The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -100,7 +103,7 @@ flaky = true # Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -114,16 +117,18 @@ flaky = true # the scenario with lemonade (the MI300X and W7900 lanes had not reported in that # run); whether lemonade serves there is not verified, and `flaky = true` tolerates # a pass on such a host. The cause on the Strix Halo Ubuntu lanes is not settled -# and varies between runs (#897), so this row names that issue rather than EAI-7423 +# and varies between runs (ROCMAI-898), so this row names that ticket rather than +# EAI-7423 # and carries `flaky = true` like its siblings. While it is in place the # scenario's own check (serve starts no background helper when no watcher is # enabled) is not exercised on the hosts it matches, and a genuine failure of that # check there would also read as a failure that was expected. `flaky` also means a -# fixed #897 is not flagged as a stale row, so delete the row by hand when #897 is +# fixed ROCMAI-898 is not flagged as a stale row, so delete the row by hand when +# ROCMAI-898 is # closed. [["serve-stop-leaves-no-background-helper"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade serving fails on the Strix Halo Ubuntu lanes, so the model never appears on /v1/models and the helper check is never reached." serve_timeout_secs = 90 flaky = true @@ -138,7 +143,7 @@ serve_timeout_secs = 90 # Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -148,16 +153,16 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the real Lemonade Linux serve stays xfail under #897; +# path works. Only the real Lemonade Linux serve stays xfail under ROCMAI-898; # the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# Bench's real-serve precondition is blocked by #897 on the same Linux hosts. +# Bench's real-serve precondition is blocked by ROCMAI-898 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -165,7 +170,7 @@ flaky = true # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 85881eb3a7..64dd0d9028 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -75,7 +75,7 @@ Feature: Model serving # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found # under bin/llamacpp// — unlike the short-recipe-name path above, whose # managed-lemonade fallback would mask the identical failure behind the - # #897 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as + # ROCMAI-898 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ba6293489..3e7a5c13c8 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -839,7 +839,7 @@ mod tests { } #[test] - fn lemonade_serve_failures_track_897_without_xfailing_mock_or_windows() { + fn lemonade_serve_failures_track_rocmai_898_without_xfailing_mock_or_windows() { let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); for id in [ "bench-load-real-serve", @@ -852,7 +852,7 @@ mod tests { assert!( matches!( resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), - Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "#897" + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCMAI-898" ), "{id}" ); @@ -927,7 +927,7 @@ mod tests { assert!( matches!( resolve(&scenario, &linux_lemonade, &matrix, Included::default()), - Expectation::ExpectXfail { bug, .. } if bug == "#897" + Expectation::ExpectXfail { bug, .. } if bug == "ROCMAI-898" ), "{id} on a real GPU host", ); @@ -2356,8 +2356,8 @@ flaky = true /// serve-40 serves through the same lemonade managed serve as the other /// lemonade serve rows, so its row is scoped the same way: xfail on a /// lemonade Linux host, with a 90s serve wait, and not an expected failure - /// on native Windows. The row must stay `flaky` (a fixed #897 would - /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name #897. + /// on native Windows. The row must stay `flaky` (a fixed ROCMAI-898 would + /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name ROCMAI-898. #[test] fn serve_40_row_is_scoped_like_the_other_lemonade_serve_rows() { let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); @@ -2371,7 +2371,7 @@ flaky = true let entries = m.entries_for(id); assert_eq!(entries.len(), 1); assert!(entries[0].flaky); - assert_eq!(entries[0].bug, "#897"); + assert_eq!(entries[0].bug, "ROCMAI-898"); } /// The row's 90s only matters if the harness hands it to the serve steps: From 4293a306352a7cd5efd77bdb14e0063d33e35a79 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:39:29 +0000 Subject: [PATCH 25/28] fix(e2e): refuse a symlink in the open itself, not in a check before it The report refused symlinked files by reading the path's metadata and then opening it, which left a window in which the path could become a link while the docs said links are refused. On unix the open now passes O_NOFOLLOW (through the libc the crate already depends on, so no new dependency), so a link is refused whenever it is there at the moment of the open, with the same "symlink not followed" message. Manifest, pins, version.txt and fingerprint reads all go through that one helper. On Windows the behaviour is unchanged: a metadata check followed by an open. A swap between the two is not caught there, and docs/testing.md says so, and that only the last path component is checked. A unix test calls the open helper directly on a link, which is what a swap after the check looks like to it, and goes red when O_NOFOLLOW is removed. Signed-off-by: Jussi Elo --- docs/testing.md | 7 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 119 ++++++++++++++---- tests/e2e-cucumber/src/serve_log.rs | 2 +- 3 files changed, 104 insertions(+), 24 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 360e5d54a1..f9e30ae723 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1066,7 +1066,12 @@ backend directory is not read; the report says how many symlinked entries it declined to follow. The same rule covers every file the report reads: the manifest, the pins file, `version.txt` and a candidate's fingerprint target are refused when the last path component is a symlink, and the report says `symlink -not followed` in place of the content. When a limit or the clock ends the search early the report +not followed` in place of the content. On unix the refusal is made by the open +itself (`O_NOFOLLOW`), so a path swapped for a link after an earlier check is still +refused; on Windows it is a metadata check followed by an open, so a swap between +the two is not caught (read-only evidence, and the swap needs write access to the +backend directory). Only the last component is checked: a link in a parent +directory is followed. When a limit or the clock ends the search early the report says which, and does not claim an empty result. When nothing is found, the report lists the searched locations and what `bin/llamacpp` holds (the first 16 names out of at most 64 entries read, with a count of those omitted), and does not diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index bd3a355b8e..fc9b743687 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -18,7 +18,7 @@ use std::process::Stdio; use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncReadExt as _}; -use crate::serve_log::{ENGINE_UNKNOWN_EVIDENCE, read_clamped}; +use crate::serve_log::{ENGINE_UNKNOWN_EVIDENCE, read_clamped_file}; const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; const PROBE_TIMEOUT: Duration = Duration::from_secs(5); @@ -175,19 +175,61 @@ impl Inventory { /// What the report says about a file it did not read because it is a symlink. const SYMLINK_NOT_FOLLOWED: &str = "symlink not followed"; -/// Whether the last component of `path` is itself a symlink (not followed). +/// Whether the last component of `path` is itself a symlink (not followed). Only +/// the non-unix [`open_no_follow`] needs it; unix refuses in the open itself. +#[cfg(not(unix))] fn is_symlink(path: &Path) -> bool { std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) } -/// [`read_clamped`], except that a path which is a symlink is refused rather than -/// followed, the same rule the directory walk applies: a link inside the backend -/// tree that points outside it must not put the outside file into the report. -fn read_regular_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { - if is_symlink(path) { - return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); +/// Whether `error` is the refusal [`open_no_follow`] makes for a symlink. +fn is_symlink_refusal(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::Other && error.to_string() == SYMLINK_NOT_FOLLOWED +} + +/// Open `path` for reading, refusing it when its last component is a symlink. +/// +/// On unix the refusal is made by the open itself (`O_NOFOLLOW`), so there is no +/// window between a check and the open in which the path could be swapped for a +/// link: a link is refused whenever it is there at the moment of the open. On +/// Windows the check is a metadata read followed by an open, so a path swapped for +/// a link between the two is not caught; the evidence is read-only and the swap +/// needs write access to the backend directory, so that is accepted rather than +/// reached for with platform calls this harness does not otherwise make. Only +/// the last component is checked on either platform: a link in a parent directory +/// is followed. +fn open_no_follow(path: &Path) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .map_err(|error| { + if error.raw_os_error() == Some(libc::ELOOP) { + std::io::Error::other(SYMLINK_NOT_FOLLOWED) + } else { + error + } + }) } - read_clamped(path, max) + #[cfg(not(unix))] + { + if is_symlink(path) { + return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); + } + std::fs::File::open(path) + } +} + +/// [`read_clamped`] on a file opened with [`open_no_follow`]: a link inside the +/// backend tree that points outside it must not put the outside file into the +/// report, the same rule the directory walk applies. +fn read_regular_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = open_no_follow(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) } /// Answers "has the inventory's time run out?". Injected so a test can drive the @@ -296,15 +338,19 @@ fn inspect_candidate(server: &Path, deadline: Instant) -> String { } if let Some(parent) = server.parent() { let version = parent.join("version.txt"); - if is_symlink(&version) { - let _ = writeln!(report, "{}: {SYMLINK_NOT_FOLLOWED}", version.display()); - } else if let Ok((bytes, note)) = read_clamped(&version, MAX_EVIDENCE_BYTES) { - let _ = writeln!( - report, - "{}: {}{note}", - version.display(), - String::from_utf8_lossy(&bytes) - ); + match read_regular_clamped(&version, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + Err(error) if is_symlink_refusal(&error) => { + let _ = writeln!(report, "{}: {SYMLINK_NOT_FOLLOWED}", version.display()); + } + Err(_) => {} } } report @@ -630,10 +676,7 @@ fn fingerprint_with_limit(server: &Path, deadline: Instant, limit: u64) -> std:: describe_hash_limit(limit) )) }; - if is_symlink(server) { - return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); - } - let mut file = std::fs::File::open(server)?; + let mut file = open_no_follow(server)?; // Do not hash an arbitrarily large corrupt cache entry while reporting a // failure. Never label a partial hash as SHA256. if file.metadata()?.len() > limit { @@ -1185,6 +1228,38 @@ mod tests { assert_eq!(found.dirs_visited, 2, "{found:?}"); } + /// The refusal is made by the open itself, not by a check before it, so a path + /// that becomes a link after any check is still refused. The helper is called + /// directly on a link, which is what a swap after the metadata check looks like + /// to it; removing `O_NOFOLLOW` makes the open succeed and turns this red. A + /// regular file still opens, and a missing one reports its own error, not a + /// refusal. (Unix only: on Windows the check and the open are separate steps.) + #[cfg(unix)] + #[test] + fn the_open_itself_refuses_a_symlink() { + use std::io::Read as _; + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let real = dir.path().join("real"); + std::fs::write(&real, "OUTSIDE").unwrap(); + let link = dir.path().join("link"); + symlink(&real, &link).unwrap(); + + let error = open_no_follow(&link).unwrap_err(); + assert!(is_symlink_refusal(&error), "{error}"); + assert_eq!(error.to_string(), SYMLINK_NOT_FOLLOWED); + + let mut text = String::new(); + open_no_follow(&real) + .unwrap() + .read_to_string(&mut text) + .unwrap(); + assert_eq!(text, "OUTSIDE"); + let missing = open_no_follow(&dir.path().join("missing")).unwrap_err(); + assert!(!is_symlink_refusal(&missing), "{missing}"); + assert_eq!(missing.kind(), std::io::ErrorKind::NotFound); + } + /// Every file the report reads from the backend tree is refused when it is a /// symlink, and the report says so, the way the walk reports a link it did not /// enter. A real `llama-server` sits beside a `version.txt` that links to a file diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index e97d62164d..90d7573ecc 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -179,7 +179,7 @@ pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, S /// "fits" with the surplus dropped without a word, so the read falls back to the /// head and tail of the file as it is now and the note says it grew. A file that /// shrank instead is returned as read, with a note that it changed. -fn read_clamped_file( +pub(crate) fn read_clamped_file( mut file: std::fs::File, len: u64, max: u64, From 198e3852088c812afead986d3decf8fc1bd0dab2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:42:33 +0000 Subject: [PATCH 26/28] fix(e2e): report a truncated candidate list only when it is truncated, and say the budget is out once Two cosmetic lies in the Lemonade report. The "truncated after 8 candidates" note was printed when exactly eight candidates existed, because the search stopped on reaching the cap and could not tell whether a ninth existed. And "remaining evidence skipped: evidence budget exhausted" was printed twice when the search ended on the clock and the candidate loop then found the budget gone too. The search now keeps eight and stops on finding a ninth, so the note appears only when there is something not listed, and exactly eight is a complete list. The budget line is said once. Both have tests, and each turns red when the old behaviour returns. docs/testing.md states the ninth-candidate rule. Signed-off-by: Jussi Elo --- docs/testing.md | 5 +- .../e2e-cucumber/src/lemonade_diagnostics.rs | 97 ++++++++++++++++--- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f9e30ae723..8416267c6f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1057,8 +1057,9 @@ under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of `bin/llamacpp`, down to three directory levels below each; the cpu backend is not searched. The search is bounded as a whole, not per directory: it reads at most 128 directories and 2048 directory entries across all four backend directories, -stops as soon as it has eight candidates, and stops when the inventory's time is -up, checking the clock at every directory and entry. Siblings are visited in +stops as soon as it finds a ninth candidate (it lists the first eight and says the +list is cut; exactly eight is not reported as cut), and stops when the +inventory's time is up, checking the clock at every directory and entry. Siblings are visited in sorted order, so a search that finishes inside its limits is deterministic. Symlinks are never followed: only real directories are entered and only a regular file named `llama-server` counts, so a link cycle ends and a link out of the diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index fc9b743687..670951de9b 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -415,12 +415,16 @@ async fn take_inventory_with( ); } let mut servers = Vec::new(); + // Said once: when the search itself ended on the clock the line is already in. + let budget_noted = matches!(discovery.stop, Some(Stop::Deadline)); for server in &discovery.candidates { if Instant::now() >= deadline { - let _ = writeln!( - report, - "remaining evidence skipped: evidence budget exhausted" - ); + if !budget_noted { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } return Inventory { report, servers }; } let _ = writeln!( @@ -441,7 +445,7 @@ async fn take_inventory_with( let _ = writeln!( report, "backend inventory truncated after {MAX_CANDIDATES} candidates \ - (the search stops there; more may exist)" + (a further one was found and is not listed)" ); } Inventory { report, servers } @@ -471,7 +475,8 @@ struct SearchLimits { dirs: usize, /// Directory entries read, in total. entries: usize, - /// Candidates collected; the search stops as soon as it has this many. + /// Candidates kept. The search stops as soon as it finds one more than this, + /// which is how it knows the list is cut; the extra one is dropped. candidates: usize, } @@ -537,6 +542,7 @@ fn discover_candidates( break; } } + search.out.candidates.truncate(limits.candidates); search.out } @@ -570,7 +576,9 @@ impl Search<'_> { if std::fs::symlink_metadata(&direct).is_ok_and(|meta| meta.is_file()) { self.out.candidates.push(direct); } - if self.out.candidates.len() >= self.limits.candidates { + // One more than the cap: finding it is what shows the list is cut, so + // exactly `candidates` of them is a complete list and not a truncated one. + if self.out.candidates.len() > self.limits.candidates { self.stop(Stop::Candidates); return; } @@ -1171,9 +1179,10 @@ mod tests { ); } - /// The candidate cap ends the walk as soon as it is reached, instead of the walk - /// finishing and the list being cut afterwards: with twenty candidate - /// directories only the root and the first eight are entered, in sorted order. + /// The candidate cap ends the walk as soon as it is certain the list is cut, + /// instead of the walk finishing and the list being cut afterwards: with twenty + /// candidate directories only the root and the first nine (eight kept, one more + /// to show there are further ones) are entered, in sorted order. #[test] fn the_candidate_cap_stops_the_walk_when_it_is_reached() { let root = tempfile::tempdir().unwrap(); @@ -1184,7 +1193,73 @@ mod tests { let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); assert_eq!(found.stop, Some(Stop::Candidates)); assert_eq!(found.candidates, wanted[..MAX_CANDIDATES]); - assert_eq!(found.dirs_visited, 1 + MAX_CANDIDATES, "{found:?}"); + assert_eq!(found.dirs_visited, 1 + MAX_CANDIDATES + 1, "{found:?}"); + } + + /// Exactly the cap is a complete list, not a truncated one: no stop, and the + /// report prints no truncation note. A ninth candidate makes it truncated, and + /// then the note is printed. Counting the cap itself as "truncated" turns the + /// first half red. + #[tokio::test] + async fn exactly_the_cap_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + for i in 0..MAX_CANDIDATES { + touch_server(&backend.join(format!("{i:02}"))); + } + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.candidates.len(), MAX_CANDIDATES); + let report = evidence(&write_manifest(root.path())).await; + assert!(!report.contains("truncated"), "{report}"); + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + + touch_server(&backend.join(format!("{MAX_CANDIDATES:02}"))); + let report = evidence(&write_manifest(root.path())).await; + assert!( + report.contains("backend inventory truncated after 8 candidates"), + "{report}" + ); + } + + /// When the search ends on the clock after finding candidates, and the hashing + /// deadline has passed too, "remaining evidence skipped" is said once, not once + /// by the search and again by the candidate loop. + #[tokio::test] + async fn the_budget_exhausted_line_is_printed_once() { + let root = tempfile::tempdir().unwrap(); + for backend in BACKEND_DIRS { + touch_server(&root.path().join("bin/llamacpp").join(backend)); + } + let calls = std::sync::atomic::AtomicUsize::new(0); + let calls = std::sync::Arc::new(calls); + let counter = calls.clone(); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + .checked_sub(Duration::from_secs(1)) + .expect("a second before now"), + SEARCH_LIMITS, + std::sync::Arc::new(move || { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }), + ) + .await; + let report = &inventory.report; + assert_eq!( + report + .matches("remaining evidence skipped: evidence budget exhausted") + .count(), + 1, + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); } /// A symlink is neither entered nor read as a candidate. The tree holds a link From 2e398267782ca2ddb7efe9f878f7142ce37b0b72 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:45:58 +0000 Subject: [PATCH 27/28] test(e2e): do not fail the probe kill test when the helper child starts slowly The kill and reap test lets a helper child run for a fixed 5 seconds and reads its pid from the probe's report. On a loaded runner a cold test binary can take longer than that to start and print the pid, which failed the test for a slow start and not for anything the probe did. An attempt whose child printed no pid before the timeout is now retried with a longer timeout (20 seconds, then 60), and only an attempt that did report a pid is asserted on, so the generous waits matter only when the start is slow and a healthy run still takes 5 seconds. With a child that takes 7 seconds to start the test passes on the second attempt, and with the retries removed it fails. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 60 +++++++++++++------ 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 670951de9b..83c16111f1 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -1934,31 +1934,53 @@ esac command } + /// How long each attempt at the timeout test lets the helper child run before the + /// probe's timeout fires. The first is short, so a healthy run is quick; the + /// later ones are generous and matter only when the child was too slow to start + /// and print its pid inside the earlier one (a loaded runner, a cold test + /// binary), which is a slow start and not a failure of the probe. + const CHILD_START_ATTEMPTS: [u64; 3] = [5, 20, 60]; + /// One run of the timeout path against a helper child that libtest runs with /// `test_threads` test threads: the child is still running when the timeout /// fires, so the report must say so, return promptly, and leave no process /// behind. `process_is_running` is false only once the child has been killed /// and reaped, so a probe that merely abandoned it turns this red. + /// + /// The pid is read back from the report, so an attempt whose child had not + /// started before the timeout (no pid line) is retried with a longer timeout + /// rather than failed; only an attempt that did report a pid is asserted on. async fn assert_timeout_kills_and_reaps(test_threads: &str) { - let start = Instant::now(); - let mut command = probe_child_command("sleep"); - command.env("RUST_TEST_THREADS", test_threads); - let report = run_probe(command, Duration::from_secs(5)).await; - assert!(report.starts_with("probe timed out after 5s"), "{report}"); - assert!(!report.contains("kill/reap failed"), "{report}"); - assert!(start.elapsed() < Duration::from_secs(20), "{report}"); - let pid = report - .lines() - .find_map(|line| line.strip_prefix("pid:")) - .unwrap_or_else(|| { - panic!( - "the child did not report its pid before the timeout \ - (test threads {test_threads}): {report}" - ) - }) - .parse::() - .unwrap(); - assert!(!rocm_core::process_is_running(pid), "{report}"); + let mut last = String::new(); + for seconds in CHILD_START_ATTEMPTS { + let start = Instant::now(); + let mut command = probe_child_command("sleep"); + command.env("RUST_TEST_THREADS", test_threads); + let report = run_probe(command, Duration::from_secs(seconds)).await; + assert!( + report.starts_with(&format!("probe timed out after {seconds}s")), + "{report}" + ); + assert!(!report.contains("kill/reap failed"), "{report}"); + assert!( + start.elapsed() < Duration::from_secs(seconds + 15), + "{report}" + ); + let Some(pid) = report + .lines() + .find_map(|line| line.strip_prefix("pid:")) + .map(|pid| pid.parse::().unwrap()) + else { + last = report; + continue; + }; + assert!(!rocm_core::process_is_running(pid), "{report}"); + return; + } + panic!( + "the child never reported its pid, even with {CHILD_START_ATTEMPTS:?} seconds \ + (test threads {test_threads}): {last}" + ); } /// The timeout path on every platform, with the child run both on one test From a577e0e32509f4276dba7d3b74a167278a0c0426 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 11:16:14 +0000 Subject: [PATCH 28/28] test(e2e): keep the helper child alive past the longest probe attempt The kill and reap test retries with timeouts of 5, 20 and 60 seconds, but the helper child exited by itself after 30. If the first two attempts timed out before the child printed its pid and the last one started it, the child exited normally before the 60 second timeout, so the probe saw a normal exit, not a timeout, and the test failed for a correct probe. Reproduced with a child that takes 25 seconds to start: the third attempt reports "exit status: 0". The helper's lifetime is now derived from the attempt timeouts (the longest plus a 60 second margin), so they cannot drift apart, and a test pins that the child outlives every attempt by at least 30 seconds. With the same 25 second start the test now passes on the third attempt; with a literal lifetime or no margin the pinning test fails. A child that takes 7 seconds to start still passes on the second attempt and fails with the retries removed. The other helper lifetimes in the file (30 second sleeps against 1 second timeouts, an immediate exit against a 60 second timeout) have no such pair. Signed-off-by: Jussi Elo --- .../e2e-cucumber/src/lemonade_diagnostics.rs | 37 ++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs index 83c16111f1..5d4eed66ab 100644 --- a/tests/e2e-cucumber/src/lemonade_diagnostics.rs +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -1913,7 +1913,7 @@ esac // line, so start a fresh one: the parent reads the first line that // begins with the marker. println!("\npid:{}", std::process::id()); - std::thread::sleep(Duration::from_secs(30)); + std::thread::sleep(Duration::from_secs(CHILD_LIFETIME_SECS)); } Ok("exit") => println!("child-ran"), _ => {} @@ -1941,6 +1941,41 @@ esac /// binary), which is a slow start and not a failure of the probe. const CHILD_START_ATTEMPTS: [u64; 3] = [5, 20, 60]; + /// How much longer than the longest attempt the helper child stays alive. The + /// child must still be running when the probe's timeout fires on any attempt: + /// one that had exited by itself would be a normal exit, not a timeout, and the + /// test would fail for a correct probe. + const CHILD_LIFETIME_MARGIN_SECS: u64 = 60; + + /// The helper child's lifetime, derived from the attempt timeouts so the two + /// cannot drift apart. + const CHILD_LIFETIME_SECS: u64 = { + let mut longest = 0; + let mut at = 0; + while at < CHILD_START_ATTEMPTS.len() { + if CHILD_START_ATTEMPTS[at] > longest { + longest = CHILD_START_ATTEMPTS[at]; + } + at += 1; + } + longest + CHILD_LIFETIME_MARGIN_SECS + }; + + /// The helper child outlives every attempt's timeout by at least half a minute + /// (the margin is asserted as a literal so shrinking the constant fails), so the + /// timeout, kill and reap path is what ends it on each retry. Giving the child a + /// literal lifetime that a longer attempt can overtake (it was 30 seconds against + /// a 60-second attempt) turns this red. + #[test] + fn the_helper_child_outlives_every_attempt_deadline() { + for attempt in CHILD_START_ATTEMPTS { + assert!( + CHILD_LIFETIME_SECS >= attempt + 30, + "the child lives {CHILD_LIFETIME_SECS}s but an attempt waits {attempt}s" + ); + } + } + /// One run of the timeout path against a helper child that libtest runs with /// `test_threads` test threads: the child is still running when the timeout /// fires, so the report must say so, return promptly, and leave no process