diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..7bdd1874f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1284,6 +1284,7 @@ dependencies = [ "rocm-core", "serde", "serde_json", + "sha2", "shlex", "tempfile", "tokio", diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 15840e56e9..7d526244e4 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -194,6 +194,21 @@ impl AppPaths { self.engine_dir(engine).join("manifests") } + /// The manifest an engine's installer writes for its installed runtime. + /// + /// One definition for the writer (the engine) and for every reader of the + /// file, so a reader cannot drift to a different name: a reader that + /// misses it degrades to "no manifest", which is indistinguishable from an + /// engine that was never installed. + /// + /// This is the engine's own install manifest, not an entry in the runtime + /// registry (that is `apps/rocm`'s `runtime_manifest_path`), and the name keeps + /// the two apart: the registry has a text guard that treats any file spelling + /// that helper's name and writing a file as a second registry writer. + pub fn engine_install_manifest_path(&self, engine: &str) -> PathBuf { + self.engine_manifests_dir(engine).join("runtime.json") + } + pub fn engine_state_dir(&self, engine: &str) -> PathBuf { self.engine_dir(engine).join("state") } @@ -355,6 +370,15 @@ mod tests { paths.data_dir.join("automations").join("proposals.jsonl") ); } + #[test] + fn engine_install_manifest_lives_in_the_engine_manifests_dir() { + let (_root, paths) = temp_app_paths("engine-runtime-manifest-path"); + assert_eq!( + paths.engine_install_manifest_path("lemonade"), + paths.engine_manifests_dir("lemonade").join("runtime.json") + ); + } + #[test] fn with_managed_root_keeps_reprovisioning_flat_from_runtime_leaf() { let data_root = PathBuf::from("/tmp/rocm-cli-reprovision"); diff --git a/docs/testing.md b/docs/testing.md index d1ba095afa..8416267c6f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1041,6 +1041,111 @@ Docker exercises AIM serve. The `e2e-gpu-nightly-strix` job logs whether Docker is reachable ("Report whether Docker is reachable"); check that line before relying on this coverage. +### Lemonade managed-serve failure evidence + +On a failed serve, the shared serve steps preserve the full service log under +`service-logs/` in the E2E results artifact before stopping the service. The engine +is taken from `--engine X` or `--engine=X`, or else from the printed serve plan; if +neither names one, the report says `engine unknown: backend evidence skipped`. A +value that starts with `--` (as in `serve m --engine --managed`) is another flag, not +an engine, and counts as no engine named. +Lemonade failures also include the installed runtime manifest's version/backend, configured +`backend_versions.json`, candidate GPU `llama-server` paths, SHA-256 fingerprints +and `version.txt`, +and each candidate's `--version` and `--help` output. Candidates are searched +under the `rocm-stable`, `rocm-nightly`, `rocm` and `vulkan` directories of +`bin/llamacpp`, down to three directory levels below each; the cpu backend is not +searched. The search is bounded as a whole, not per directory: it reads at most +128 directories and 2048 directory entries across all four backend directories, +stops as soon as it finds a ninth candidate (it lists the first eight and says the +list is cut; exactly eight is not reported as cut), and stops when the +inventory's time is up, checking the clock at every directory and entry. Siblings are visited in +sorted order, so a search that finishes inside its limits is deterministic. +Symlinks are never followed: only real directories are entered and only a regular +file named `llama-server` counts, so a link cycle ends and a link out of the +backend directory is not read; the report says how many symlinked entries it +declined to follow. The same rule covers every file the report reads: the +manifest, the pins file, `version.txt` and a candidate's fingerprint target are +refused when the last path component is a symlink, and the report says `symlink +not followed` in place of the content. On unix the refusal is made by the open +itself (`O_NOFOLLOW`), so a path swapped for a link after an earlier check is still +refused; on Windows it is a metadata check followed by an open, so a swap between +the two is not caught (read-only evidence, and the swap needs write access to the +backend directory). Only the last component is checked: a link in a parent +directory is followed. When a limit or the clock ends the search early the report +says which, and does not claim an empty result. When nothing is found, the report +lists the searched locations and what `bin/llamacpp` holds (the first 16 names +out of at most 64 entries read, with a count of those omitted), and does not +guess at the cause: an interrupted download, a layout outside the search and a +build that was never installed look the same. The manifest and pin reads, the +search and each candidate's fingerprint run on blocking threads, never on an +async worker, and the search carries its own deadline because a timeout around a +blocking task would only drop its handle. +At most eight candidates +are hashed and probed, with two read-only probes per candidate. Each probe retains +up to 32 KiB of stdout and 32 KiB of stderr; only streams that discard bytes are +marked as truncated. Across sixteen probes this is up to 1 MiB of retained raw +output, plus metadata and formatting (invalid UTF-8 bytes may expand when rendered). +Each probe has a five-second timeout. The inventory has a 20-second budget that +starts when it is entered and covers the manifest and pin reads, the search and +the hashing; the probes have their own 20-second budget, which starts +when the service stop returns; hashing runs off the async executor, rejects files +over 512 MiB, and checks its deadline between 64 KiB reads, so memory stays at +one buffer whatever the size. The cap sits well above a dynamically linked +`llama-server`: the `rocm-stable` build b12109 on the Strix Halo Ubuntu lanes +was fingerprinted under the former 32 MiB cap, so current builds fit with room to +spare. The larger cap is margin for a statically linked multi-target build, +which is an assumption about future builds, not a measured size, and a larger +file is a finding in its own right. A candidate over the size cap gets no fingerprint, and +its entry says so with `sha256 unavailable: executable exceeds 512 MiB hash +limit`; it is still listed and probed. Filesystem +operations and process cleanup are not preemptible, so this is not a hard +wall-clock limit on the entire report. +A candidate or configured build pin is **not proof** of which binary the router +launched; a failed probe is retained as evidence, not treated as an identity. +Probes inherit the harness environment rather than reconstructing the engine's +managed library paths, so a probe loader error alone does not establish why the +serve failed. The report reads files first and executes a backend second. The +manifest, pins, candidate paths, fingerprints and `version.txt` are read once the +serve has exited non-zero or its readiness wait has timed out and the log has +been read and archived, but before the service is stopped, so they describe the state being +diagnosed. The `--version` and `--help` probes run after the stop returns, in the +report's "probes, run after the service stop was attempted" section: llama.cpp +builds commonly initialise a GPU backend even for `--version`, and doing that next +to a still-running router on a shared-memory device could change the state being +diagnosed. A stop that fails or finds no record can leave the router running; the +report's `stop` section says so, and the probes still run. No lane has shown +whether a probe initialises a GPU backend, which is why they are kept off the +router's side of the stop. The order is fixed in one function, +`backend_evidence_around_stop`, whose unit tests make the stop delete the candidate +to show the fingerprint was read before it and the probes came after it. Nothing +before the stop may keep it from running: a serve with no isolated scenario root +has no manifest to look for, so the report says `backend evidence unavailable: +the scenario has no isolated root` (and the stop's own status says it was not +attempted), and a panic while looking the manifest up is caught and reported the +same way, so the stop runs exactly once and the failure report is not lost. + +The probe's timeout, kill and reap path is tested on every platform, Windows +included: the unit tests re-run the test binary itself as the child (an ignored +helper test selected by an environment variable), let the probe time out, and +require that the child is gone afterwards. The shell-script `llama-server` +stand-ins used by the other probe tests exist on unix only. Not covered anywhere +is the `kill/reap failed` branch, which needs an operating system that refuses to +kill a child it just started. + +Use the evidence to distinguish backend startup, download, and router readiness +failures; ROCMAI-898 tracks the Strix Halo Ubuntu investigation (first tracked +as [issue #897](https://github.com/ROCm/rocm-cli/issues/897), which is closed). A +backend's ready line does not establish that the router serves the requested +model, and a readiness timeout alone does not establish a shutdown. Diagnostics do not change the CLI's asynchronous +managed-launch contract or imply a backend fix. + +Exercise the diagnostics and expectation scoping without a GPU: + +```bash +cargo test -p e2e-cucumber --lib lemonade +``` + ## Runtime Selection And Activation List registered managed or read-only runtimes and their exact side-by-side keys: @@ -2029,7 +2134,7 @@ exit) and `serve-40` in (GPU lane: serving with no watcher enabled starts no helper; on a Linux host that runs it with lemonade it is an expected failure in [expectations.toml](../tests/e2e-cucumber/expectations.toml) while lemonade -serving on the Strix Halo Ubuntu lanes is broken (#897), so the helper check is +serving on the Strix Halo Ubuntu lanes is broken (ROCMAI-898), so the helper check is not reached on those lanes. The Strix Halo Windows lane runs it and passes, and the row also matches other Linux lemonade hosts, where a pass is tolerated). diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index e0fa931d11..5c67cbcad4 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -196,7 +196,7 @@ where } fn manifest_path(paths: &AppPaths) -> PathBuf { - paths.engine_manifests_dir(ENGINE_NAME).join("runtime.json") + paths.engine_install_manifest_path(ENGINE_NAME) } fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { @@ -573,6 +573,43 @@ mod tests { #[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; + /// The e2e harness reads this manifest back to describe a failed serve + /// (`tests/e2e-cucumber/src/lemonade_diagnostics.rs`), by the same path + /// helper and the field names asserted here. A rename on either side would + /// otherwise show up only as `unavailable:` in a report from the real lane. + #[test] + fn written_manifest_is_where_and_how_the_e2e_failure_report_reads_it() { + let dir = scratch_dir("manifest-path-and-fields"); + let paths = AppPaths { + config_dir: dir.join("config"), + data_dir: dir.join("data"), + cache_dir: dir.join("cache"), + }; + let manifest = LemonadeInstallManifest { + env_id: "lemonade-embeddable-fixture".to_owned(), + version: "fixture".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("runtime/lemond"), + lemonade: dir.join("runtime/lemonade"), + backend_recipe: LLAMACPP_RECIPE.to_owned(), + backend_name: ROCM_BACKEND_NAME.to_owned(), + installed_at_unix_ms: 1, + }; + write_manifest(&paths, &manifest).unwrap(); + + let path = paths.engine_install_manifest_path("lemonade"); + assert_eq!(manifest_path(&paths), path); + let written: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + assert_eq!(written["version"], "fixture"); + assert_eq!(written["backend_recipe"], LLAMACPP_RECIPE); + assert_eq!(written["backend_name"], ROCM_BACKEND_NAME); + assert_eq!( + written["runtime_dir"], + serde_json::json!(manifest.runtime_dir) + ); + fs::remove_dir_all(&dir).ok(); + } + #[test] fn embeddable_package_matches_runtime_os() { let version = rocm_deps::LEMONADE_VERSION; diff --git a/scripts/xfail_expectations_hint.py b/scripts/xfail_expectations_hint.py index 7a8439f6ce..ff38838d13 100644 --- a/scripts/xfail_expectations_hint.py +++ b/scripts/xfail_expectations_hint.py @@ -143,6 +143,11 @@ def emit(hits: dict[str, list[str]]) -> None: when = {} bug = "#123" reason = "example" + +[["serve-lemonade-inference"]] +when = { effective_engine = "lemonade", os = "linux" } +bug = "ROCMAI-898" +reason = "example" """ @@ -188,6 +193,15 @@ def check(title="", body="", commits=""): assert hits == {"EAI-7333": ["serve-vllm-inference"]}, hits print("self-test: refs read from commit messages ok") + # A row keyed to an open tracker id follows that id, and an id that is no + # longer in any row (a closed issue the rows were repointed away from) does + # not match: the hint reads the ids out of the matrix, it hard-codes none. + hits = check(body="Refs ROCMAI-898") + assert hits == {"ROCMAI-898": ["serve-lemonade-inference"]}, hits + hits = check(body="Refs #897") + assert hits == {}, hits + print("self-test: hint follows the id in the matrix (ROCMAI-898 yes, #897 no) ok") + print("xfail hint self-test: ok") diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 48fe7bbeac..8a46d60c7c 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -61,8 +61,9 @@ rand.workspace = true reqwest = { version = "0.13", features = ["json"] } serde.workspace = true serde_json.workspace = true +sha2.workspace = true tempfile = "3" -tokio.workspace = true +tokio = { workspace = true, features = ["process", "io-util"] } toml = "1.1" # `ServeDir` backs the install scenario's loopback download server (see # `src/loopback_http.rs`), so the E2E suite does not hand-roll HTTP framing, diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 340bc546d9..fcb0b52a4e 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -58,60 +58,53 @@ when = {} bug = "EAI-7219" reason = "A vLLM serve of a short name does not resolve to lemonade's full model name: it either stops before printing a serve plan or prints the alias unexpanded." -# EAI-7423 was INTERMITTENT on Strix Halo Ubuntu as of 2026-08-05, not fixed: -# which of its scenarios pass varied run to run (run 30995473578: 2 XPASS; run -# 30993213188: 3 XPASS; run 30939967076: all 6). A deterministic-XPASS row fails -# that lane on unrelated PRs, so the EAI-7423 lemonade-linux rows below carry -# `flaky = true` — either outcome is tolerated while the bug stays open. The flag -# is per-row and is NOT derived from `bug`, so a new EAI-7423 row needs it added -# explicitly. Revisit under EAI-7853 before deleting any of them: which lemonade -# build a given run used cannot currently be read back from this lane, so an -# XPASS streak here is not by itself evidence that the bug is fixed. +# ROCMAI-898 tracks multiple Strix Halo Ubuntu Lemonade serve signatures (first +# tracked as GitHub issue ROCm/rocm-cli#897, since closed; ROCMAI-898 is the open +# ticket and what the rows below name): +# the child rejects --no-mmap (41984921053), backend readiness on :8001 without +# the model appearing on :11435 (41984503055), and backend download failure +# (41986772728). A backend ready line does not prove a later shutdown. +# EAI-7423 described only the earlier ready-then-unavailable symptom. +# Retain flaky=true because historical runs also passed intermittently; do not +# infer a fix from a single XPASS. The flag is per-row and is NOT derived from +# `bug`, so a new ROCMAI-898 row needs `flaky = true` added explicitly. +# Failed serves now capture configured pins, candidate executable paths and +# fingerprints before stopping the service, and each candidate's --version/--help +# after it. +# Revisit under EAI-7853 before deleting any of these rows: which lemonade build +# a given run used could not be read back from this lane, and until a failed run +# there has carried that evidence, an XPASS streak is not by itself evidence that +# the bug is fixed. # -# On a lemonade-native LINUX host (Strix Halo Ubuntu) the lemonade managed serve -# loads the model and reaches ready on :8001, then is shut down ~0.08s later, so -# the endpoint never becomes reachable (EAI-7423, root-caused on the box -# 2026-07-15 — NOT the earlier "backend install hides the plan line" theory, which -# was disproven: the `engine:` line IS emitted). os=linux: on native Windows this -# scenario PASSES (run 29338475133 Strix-Windows), so it must NOT be xfail'd there. +# os=linux: native Windows passed (29338475133 Strix-Windows). [["serve-default-engine-working-endpoint"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the endpoint never becomes reachable." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade-native LINUX host the served model's endpoint drops before -# inference: the lemonade managed serve reaches ready then is shut down ~0.08s -# later (EAI-7423, root-caused on the box 2026-07-15 — not a Vulkan hang). -# os=linux: passes on native Windows (run 29338475133). +# The same serve precondition blocks inference; native Windows passed. [["serve-default-engine-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# On a lemonade LINUX host this scenario does a real lemonade serve, which reaches -# ready then shuts down immediately (EAI-7423, see serve-lemonade-inference). -# os=linux: on native Windows lemonade serve works, so this PASSES there (verified -# run 29338475133 Strix-Windows XPASS) and must not be xfail'd. +# The readiness contract depends on the same real Lemonade serve. [["serve-readiness-contract"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the ready-to-serve contract fails." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# --- EAI-7423: on a lemonade LINUX host the managed serve reaches ready then is -# shut down immediately, so inference never happens (root-caused on the box -# 2026-07-15; previously mis-attributed to the EAI-7052 Vulkan hang). os=linux: -# native Windows lemonade serve works (run 29338475133 Strix-Windows XPASS'd -# this), so scope the xfail to Linux. --- +# Explicit Lemonade serving shares the same failure on the Ubuntu lane. [["serve-lemonade-inference"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so inference never succeeds." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -124,16 +117,18 @@ flaky = true # the scenario with lemonade (the MI300X and W7900 lanes had not reported in that # run); whether lemonade serves there is not verified, and `flaky = true` tolerates # a pass on such a host. The cause on the Strix Halo Ubuntu lanes is not settled -# and varies between runs (#897), so this row names that issue rather than EAI-7423 +# and varies between runs (ROCMAI-898), so this row names that ticket rather than +# EAI-7423 # and carries `flaky = true` like its siblings. While it is in place the # scenario's own check (serve starts no background helper when no watcher is # enabled) is not exercised on the hosts it matches, and a genuine failure of that # check there would also read as a failure that was expected. `flaky` also means a -# fixed #897 is not flagged as a stale row, so delete the row by hand when #897 is +# fixed ROCMAI-898 is not flagged as a stale row, so delete the row by hand when +# ROCMAI-898 is # closed. [["serve-stop-leaves-no-background-helper"]] when = { effective_engine = "lemonade", os = "linux" } -bug = "#897" +bug = "ROCMAI-898" reason = "Lemonade serving fails on the Strix Halo Ubuntu lanes, so the model never appears on /v1/models and the helper check is never reached." serve_timeout_secs = 90 flaky = true @@ -145,16 +140,11 @@ bug = "EAI-7223" reason = "Chat requests carrying tool definitions are not accepted by the served vLLM model." serve_timeout_secs = 90 -# On a lemonade LINUX host this needs a live lemonade serve, which reaches ready -# then shuts down immediately (EAI-7423). os=linux: passes on native Windows -# lemonade (run 29354305288 XPASS'd it), consistent with the other lemonade -# serve scenarios. therock_family=gfx* scopes this to a REAL GPU host: the -# scenario now also runs on the no-GPU mock lane (backed by MockServer, not a -# real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# Chat's mock lane must remain expect-pass: only a real GPU serve is affected. [["chat-tool-definitions-accepted"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true @@ -163,27 +153,16 @@ flaky = true # scenario exercises the CLI `rocm chat --prompt` path (which correctly discovers the # model via /v1/models). Verified expect-pass on MI300X vLLM (XPASS on run 29322186691, # then 3/3 green re-runs), so there is NO vLLM xfail for this scenario — the CLI chat -# path works. Only the lemonade LINUX path stays xfail: it needs a live lemonade -# serve, which reaches ready then shuts down immediately (EAI-7423); native Windows -# lemonade serve works (run 29338475133 XPASS). therock_family=gfx* scopes this to -# a REAL GPU host: the scenario also runs on the no-GPU mock lane (MockServer, not -# a real lemonade serve), where EAI-7423 does not apply and it must expect-pass. +# path works. Only the real Lemonade Linux serve stays xfail under ROCMAI-898; +# the no-GPU MockServer path and native Windows remain expect-pass. [["chat-end-to-end-local-model"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the chat request never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true -# The GPU-backed bench scenario benchmarks a really served model, so on a -# lemonade LINUX host it inherits the same managed-serve bug as the sibling -# serve/chat rows below `serve-default-engine-working-endpoint`: the serve -# reaches ready then shuts down immediately, so this scenario's `a model is -# being served on GPU` precondition never yields a live endpoint. Scoped -# identically — os=linux only, because native Windows lemonade serve works, and -# therock_family=gfx* so it applies only on a real GPU host. `flaky = true` -# matches those rows too: which of them pass varies run to run on that lane, and -# a deterministic-XPASS row would fail the lane on unrelated PRs. +# Bench's real-serve precondition is blocked by ROCMAI-898 on the same Linux hosts. # # This does NOT weaken the coverage the bench fix relies on: the five # MockServer-backed bench scenarios (bench-01 to bench-05) run unxfail'd on @@ -191,7 +170,7 @@ flaky = true # failure reporting, the CSV columns, and the rollup-split warning. [["bench-load-real-serve"]] when = { effective_engine = "lemonade", os = "linux", therock_family = "gfx*", has_amd_gpu = true } -bug = "EAI-7423" -reason = "Lemonade managed serve reaches ready then shuts down immediately, so the benchmark never reaches a live model." +bug = "ROCMAI-898" +reason = "Lemonade managed serve never exposes the model: observed backend argument rejection, backend download failure, or backend-ready/router-unavailable." serve_timeout_secs = 90 flaky = true diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index ffbb2fd692..64dd0d9028 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -75,7 +75,7 @@ Feature: Model serving # (`serve_hf_checkpoint`), which bails explicitly if no backend binary is found # under bin/llamacpp// — unlike the short-recipe-name path above, whose # managed-lemonade fallback would mask the identical failure behind the - # unrelated EAI-7423 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as + # ROCMAI-898 xfail. Reuses the same small Qwen3-0.6B-GGUF checkpoint as # `serve-lemonade-inference` (cache-shared, no extra download) so this stays a # fast per-PR canary rather than needing the @nightly large-checkpoint path. @id:serve-hf-checkpoint-inference @requires-gpu @requires-engine:lemonade diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 01ed93ca03..3e7a5c13c8 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -838,6 +838,102 @@ mod tests { } } + #[test] + fn lemonade_serve_failures_track_rocmai_898_without_xfailing_mock_or_windows() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + "serve-lemonade-inference", + "serve-stop-leaves-no-background-helper", + ] { + let scenario = decl(&[&format!("id:{id}")]); + assert!( + matches!( + resolve(&scenario, &cap("strix-ubuntu"), &matrix, Included::default()), + Expectation::ExpectXfail { bug, flaky: true, .. } if bug == "ROCMAI-898" + ), + "{id}" + ); + assert_eq!( + matrix.serve_timeout_for(id, &cap("strix-ubuntu"), "lemonade"), + Some(90), + "{id}", + ); + for platform in ["mock", "strix-windows"] { + assert_eq!( + resolve(&scenario, &cap(platform), &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {platform}", + ); + } + } + } + + /// The rows that carry `therock_family` and `has_amd_gpu` exist so that the + /// no-GPU lane, where the same scenarios run against MockServer instead of a + /// real Lemonade serve, stays expect-pass. `cap("mock")` cannot prove that: + /// its `os_family` is `other`, so the `os = "linux"` condition excludes it + /// before either GPU guard is consulted. These hosts are Linux, so each guard + /// is the only thing standing between them and an xfail. The two partial + /// hosts isolate the guards from each other: dropping `has_amd_gpu` from a row + /// fails the host that reports only a gfx target, and dropping `therock_family` + /// fails the host that reports only a GPU. + #[test] + fn real_gpu_lemonade_rows_do_not_xfail_linux_hosts_without_a_usable_gpu() { + let matrix = Expectations::parse(include_str!("../expectations.toml")).unwrap(); + let linux_lemonade = cap("strix-ubuntu"); + let hosts = [ + ( + "linux, no GPU, no gfx target", + HostCapability { + gfx_target: None, + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, gfx target reported but no usable GPU", + HostCapability { + has_amd_gpu: false, + amd_gpu_count: Some(0), + ..linux_lemonade.clone() + }, + ), + ( + "linux, usable GPU but no gfx target reported", + HostCapability { + gfx_target: None, + ..linux_lemonade.clone() + }, + ), + ]; + for id in [ + "bench-load-real-serve", + "chat-tool-definitions-accepted", + "chat-end-to-end-local-model", + ] { + let scenario = decl(&[&format!("id:{id}")]); + for (label, host) in &hosts { + assert_eq!( + resolve(&scenario, host, &matrix, Included::default()), + Expectation::ExpectPass, + "{id} on {label}", + ); + } + // The full real-GPU host is the one that must still be xfailed. + assert!( + matches!( + resolve(&scenario, &linux_lemonade, &matrix, Included::default()), + Expectation::ExpectXfail { bug, .. } if bug == "ROCMAI-898" + ), + "{id} on a real GPU host", + ); + } + } + fn decl(tags: &[&str]) -> ScenarioDecl { ScenarioDecl::from_tags(tags) } @@ -2260,8 +2356,8 @@ flaky = true /// serve-40 serves through the same lemonade managed serve as the other /// lemonade serve rows, so its row is scoped the same way: xfail on a /// lemonade Linux host, with a 90s serve wait, and not an expected failure - /// on native Windows. The row must stay `flaky` (a fixed #897 would - /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name #897. + /// on native Windows. The row must stay `flaky` (a fixed ROCMAI-898 would + /// otherwise turn the Ubuntu lanes red as a stale XPASS) and must name ROCMAI-898. #[test] fn serve_40_row_is_scoped_like_the_other_lemonade_serve_rows() { let m = Expectations::parse(include_str!("../expectations.toml")).unwrap(); @@ -2275,7 +2371,7 @@ flaky = true let entries = m.entries_for(id); assert_eq!(entries.len(), 1); assert!(entries[0].flaky); - assert_eq!(entries[0].bug, "#897"); + assert_eq!(entries[0].bug, "ROCMAI-898"); } /// The row's 90s only matters if the harness hands it to the serve steps: diff --git a/tests/e2e-cucumber/src/lemonade_diagnostics.rs b/tests/e2e-cucumber/src/lemonade_diagnostics.rs new file mode 100644 index 0000000000..5d4eed66ab --- /dev/null +++ b/tests/e2e-cucumber/src/lemonade_diagnostics.rs @@ -0,0 +1,2116 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Read-only backend evidence for a failed Lemonade serve. +//! +//! The evidence is taken in two steps with the service stop between them (see +//! [`backend_evidence_around_stop`]): what is only read from disk (manifest, +//! pins, candidate paths, fingerprints) is collected while the failed service is +//! still in the state being diagnosed, and the `llama-server` probes, which +//! execute a backend binary, run after it has been stopped. + +use sha2::{Digest as _, Sha256}; +use std::fmt::Write as _; +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncRead, AsyncReadExt as _}; + +use crate::serve_log::{ENGINE_UNKNOWN_EVIDENCE, read_clamped_file}; + +const MAX_EVIDENCE_BYTES: u64 = 64 * 1024; +const PROBE_TIMEOUT: Duration = Duration::from_secs(5); +const MAX_CANDIDATES: usize = 8; +/// Time for the read-only inventory (pins, candidate paths, fingerprints), +/// taken before the service is stopped. +const INVENTORY_BUDGET: Duration = Duration::from_secs(20); +/// Time for the probes, started after the stop returns so a slow stop does not +/// use it up. +const PROBE_BUDGET: Duration = Duration::from_secs(20); +/// Largest executable that is fingerprinted. +/// +/// A fingerprint that is absent on the lane the report is for is a defect, so the +/// cap is set well above a dynamically linked `llama-server` (a few MiB; the HIP +/// kernels live in separate shared libraries). It is still finite: a build that +/// statically linked kernels for several GPU targets can reach hundreds of MiB, and +/// a corrupt cache entry should not be hashed at all. Hashing streams 64 KiB at a +/// time, so memory stays bounded whatever this is; the time is bounded by the +/// inventory budget, which the hashing loop checks between reads. +const MAX_HASH_BYTES: u64 = 512 * 1024 * 1024; + +#[derive(Clone, Copy)] +struct Budgets { + inventory: Duration, + probes: Duration, +} + +const DEFAULT_BUDGETS: Budgets = Budgets { + inventory: INVENTORY_BUDGET, + probes: PROBE_BUDGET, +}; + +/// Runs one `llama-server` flag and renders what came back. A trait so the order +/// of probes against the service stop is testable without executing a binary. +trait Probe: Sync { + fn run( + &self, + server: &Path, + flag: &str, + timeout: Duration, + ) -> impl std::future::Future + Send; +} + +struct RealProbe; + +impl Probe for RealProbe { + async fn run(&self, server: &Path, flag: &str, timeout: Duration) -> String { + probe(server, flag, timeout).await + } +} + +/// Collect the backend evidence for a failed serve of `engine` around the call +/// that stops the service, and return the stop's own status with it. +/// +/// `stop` always runs exactly once, whatever the engine, so a caller cannot +/// forget it or run it twice. For Lemonade the order is fixed: the read-only +/// inventory first, then `stop`, then the probes. A probe executes a backend +/// binary, and llama.cpp builds commonly initialise a GPU backend even for +/// `--version`, so running one next to a still-live router on a shared-memory +/// device could change the very state being diagnosed. Reading files cannot. +/// +/// `manifest_path` is only called for Lemonade, and nothing it does may keep the +/// stop from running: it returns `Err(reason)` when there is no manifest to look +/// for (the report then says `unavailable: `), and a panic inside it is +/// caught and reported the same way, so the stop still runs exactly once and the +/// failure report survives. The report is the one thing a failed serve has left. +/// +/// The report says `engine unknown` when no engine was determined, so a skipped +/// inventory stays visible, and returns no evidence for another engine. +pub async fn backend_evidence_around_stop( + manifest_path: impl FnOnce() -> Result + Send, + engine: Option<&str>, + stop: impl FnOnce() -> String + Send, +) -> (String, Option) { + match engine { + Some("lemonade") => { + let manifest = std::panic::catch_unwind(std::panic::AssertUnwindSafe(manifest_path)) + .unwrap_or_else(|_| Err("finding the runtime manifest failed".to_owned())); + match manifest { + Ok(manifest) => { + let (stop_status, evidence) = + evidence_around_stop(&manifest, DEFAULT_BUDGETS, &RealProbe, stop).await; + (stop_status, Some(evidence)) + } + Err(reason) => ( + stop(), + Some(format!("backend evidence unavailable: {reason}")), + ), + } + } + Some(_) => (stop(), None), + None => (stop(), Some(ENGINE_UNKNOWN_EVIDENCE.to_owned())), + } +} + +async fn evidence_around_stop( + manifest_path: &Path, + budgets: Budgets, + probe: &impl Probe, + stop: impl FnOnce() -> String + Send, +) -> (String, String) { + let inventory = take_inventory(manifest_path, budgets.inventory).await; + let stop_status = stop(); + let probes = inventory.probe_all(probe, budgets.probes).await; + (stop_status, format!("{}{probes}", inventory.report)) +} + +/// What was read from disk before the stop: the report so far, and the +/// candidates the probes will run against once the service is stopped. +struct Inventory { + report: String, + servers: Vec, +} + +impl Inventory { + const fn without_probes(report: String) -> Self { + Self { + report, + servers: Vec::new(), + } + } + + async fn probe_all(&self, probe: &impl Probe, budget: Duration) -> String { + if self.servers.is_empty() { + return String::new(); + } + let mut report = String::new(); + let _ = writeln!( + report, + "probes, run after the service stop was attempted; they inherit the harness \ + environment, not the engine's managed library environment, so a probe loader \ + failure does not establish a serving failure:" + ); + let deadline = Instant::now() + budget; + for server in &self.servers { + let _ = writeln!(report, "probe target: {}", server.display()); + for flag in ["--version", "--help"] { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + return report; + } + let output = probe.run(server, flag, remaining.min(PROBE_TIMEOUT)).await; + let _ = writeln!(report, "{flag}:\n{output}"); + } + } + report + } +} + +/// What the report says about a file it did not read because it is a symlink. +const SYMLINK_NOT_FOLLOWED: &str = "symlink not followed"; + +/// Whether the last component of `path` is itself a symlink (not followed). Only +/// the non-unix [`open_no_follow`] needs it; unix refuses in the open itself. +#[cfg(not(unix))] +fn is_symlink(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|meta| meta.file_type().is_symlink()) +} + +/// Whether `error` is the refusal [`open_no_follow`] makes for a symlink. +fn is_symlink_refusal(error: &std::io::Error) -> bool { + error.kind() == std::io::ErrorKind::Other && error.to_string() == SYMLINK_NOT_FOLLOWED +} + +/// Open `path` for reading, refusing it when its last component is a symlink. +/// +/// On unix the refusal is made by the open itself (`O_NOFOLLOW`), so there is no +/// window between a check and the open in which the path could be swapped for a +/// link: a link is refused whenever it is there at the moment of the open. On +/// Windows the check is a metadata read followed by an open, so a path swapped for +/// a link between the two is not caught; the evidence is read-only and the swap +/// needs write access to the backend directory, so that is accepted rather than +/// reached for with platform calls this harness does not otherwise make. Only +/// the last component is checked on either platform: a link in a parent directory +/// is followed. +fn open_no_follow(path: &Path) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path) + .map_err(|error| { + if error.raw_os_error() == Some(libc::ELOOP) { + std::io::Error::other(SYMLINK_NOT_FOLLOWED) + } else { + error + } + }) + } + #[cfg(not(unix))] + { + if is_symlink(path) { + return Err(std::io::Error::other(SYMLINK_NOT_FOLLOWED)); + } + std::fs::File::open(path) + } +} + +/// [`read_clamped`] on a file opened with [`open_no_follow`]: a link inside the +/// backend tree that points outside it must not put the outside file into the +/// report, the same rule the directory walk applies. +fn read_regular_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = open_no_follow(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) +} + +/// Answers "has the inventory's time run out?". Injected so a test can drive the +/// traversal's deadline without sleeping; the real one compares against an +/// [`Instant`]. `Arc` because the scan runs on a blocking thread. +type Expired = std::sync::Arc bool + Send + Sync>; + +/// Record the installed version and pins separately from executable build output: +/// a pin describes what should be installed, not what a runner actually cached. +/// +/// Every candidate is labelled as such; finding a binary does not prove that the +/// router launched it. Nothing here executes anything. No install, download, +/// model load, or cache repair is done. +/// +/// The budget starts here, at entry, and covers the manifest and pin reads, the +/// directory search and the hashing alike. +async fn take_inventory(manifest_path: &Path, budget: Duration) -> Inventory { + let deadline = Instant::now() + budget; + take_inventory_with( + manifest_path, + deadline, + SEARCH_LIMITS, + std::sync::Arc::new(move || Instant::now() >= deadline), + ) + .await +} + +/// What [`scan_runtime`] read from disk. +enum Scan { + /// The manifest could not be used; the report says why. + Unavailable(String), + Found { + /// Manifest fields and pins. + report: String, + discovery: Discovery, + /// Where an empty search looked, when it found nothing and was not cut short + /// by the clock. + empty_note: Option, + }, +} + +/// Everything the inventory reads before it hashes anything: the manifest, the +/// pins and the directory search. Blocking filesystem work, so it runs on a +/// blocking thread; it carries its own clock because a `timeout` around +/// `spawn_blocking` would only drop the join handle and leave this running. +fn scan_runtime(manifest_path: &Path, limits: SearchLimits, expired: &dyn Fn() -> bool) -> Scan { + let mut report = String::new(); + let _ = writeln!(report, "manifest: {}", manifest_path.display()); + let manifest = + match read_regular_clamped(manifest_path, MAX_EVIDENCE_BYTES).and_then(|(bytes, _)| { + serde_json::from_slice::(&bytes).map_err(std::io::Error::other) + }) { + Ok(manifest) => manifest, + Err(error) => return Scan::Unavailable(format!("{report}unavailable: {error}")), + }; + for key in ["version", "backend_recipe", "backend_name", "runtime_dir"] { + let _ = writeln!( + report, + "{key}: {}", + manifest.get(key).unwrap_or(&serde_json::Value::Null) + ); + } + let Some(runtime) = manifest + .get("runtime_dir") + .and_then(serde_json::Value::as_str) + else { + return Scan::Unavailable(format!( + "{report}backend inventory unavailable: no runtime_dir" + )); + }; + let runtime = Path::new(runtime); + let pins = runtime.join("resources/backend_versions.json"); + let _ = writeln!(report, "configured pins (not executable identity):"); + match read_regular_clamped(&pins, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!(report, "{}{note}", String::from_utf8_lossy(&bytes)); + } + Err(error) => { + let _ = writeln!(report, "{}: {error}", pins.display()); + } + } + let discovery = discover_candidates(runtime, limits, expired); + // "Nothing found" is only a finding when the search looked everywhere; one cut + // short by the clock or a limit reports that instead (see `take_inventory_with`). + // A candidate-cap stop has candidates by definition, so it never gets here. + let empty_note = (discovery.candidates.is_empty() && discovery.stop.is_none()) + .then(|| empty_search_note(runtime)); + Scan::Found { + report, + discovery, + empty_note, + } +} + +/// One candidate's fingerprint and `version.txt`, rendered. Blocking reads, run on +/// a blocking thread with the inventory's deadline checked between hash reads. +fn inspect_candidate(server: &Path, deadline: Instant) -> String { + let mut report = String::new(); + match fingerprint(server, deadline) { + Ok(hash) => { + let _ = writeln!(report, "sha256: {hash}"); + } + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + if let Some(parent) = server.parent() { + let version = parent.join("version.txt"); + match read_regular_clamped(&version, MAX_EVIDENCE_BYTES) { + Ok((bytes, note)) => { + let _ = writeln!( + report, + "{}: {}{note}", + version.display(), + String::from_utf8_lossy(&bytes) + ); + } + Err(error) if is_symlink_refusal(&error) => { + let _ = writeln!(report, "{}: {SYMLINK_NOT_FOLLOWED}", version.display()); + } + Err(_) => {} + } + } + report +} + +async fn take_inventory_with( + manifest_path: &Path, + deadline: Instant, + limits: SearchLimits, + expired: Expired, +) -> Inventory { + let scan_path = manifest_path.to_owned(); + let scan_clock = expired.clone(); + let scan = + tokio::task::spawn_blocking(move || scan_runtime(&scan_path, limits, scan_clock.as_ref())) + .await; + let (mut report, discovery, empty_note) = match scan { + Ok(Scan::Found { + report, + discovery, + empty_note, + }) => (report, discovery, empty_note), + Ok(Scan::Unavailable(report)) => return Inventory::without_probes(report), + Err(error) => { + return Inventory::without_probes(format!( + "manifest: {}\nunavailable: inventory scan failed: {error}", + manifest_path.display() + )); + } + }; + if let Some(note) = empty_note { + let _ = writeln!(report, "{note}"); + } + match discovery.stop { + Some(Stop::Deadline) => { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } + Some(Stop::Directories) => { + let _ = writeln!( + report, + "candidate search stopped: {} directories visited (limit)", + discovery.dirs_visited + ); + } + Some(Stop::Entries) => { + let _ = writeln!( + report, + "candidate search stopped: {} directory entries read (limit)", + discovery.entries_read + ); + } + Some(Stop::Candidates) | None => {} + } + if discovery.symlinks_skipped > 0 { + let _ = writeln!( + report, + "{} symlinked entries under the backend directories were not followed", + discovery.symlinks_skipped + ); + } + let mut servers = Vec::new(); + // Said once: when the search itself ended on the clock the line is already in. + let budget_noted = matches!(discovery.stop, Some(Stop::Deadline)); + for server in &discovery.candidates { + if Instant::now() >= deadline { + if !budget_noted { + let _ = writeln!( + report, + "remaining evidence skipped: evidence budget exhausted" + ); + } + return Inventory { report, servers }; + } + let _ = writeln!( + report, + "candidate (not proof of selection): {}", + server.display() + ); + servers.push(server.clone()); + let path = server.clone(); + match tokio::task::spawn_blocking(move || inspect_candidate(&path, deadline)).await { + Ok(lines) => report.push_str(&lines), + Err(error) => { + let _ = writeln!(report, "sha256 unavailable: {error}"); + } + } + } + if matches!(discovery.stop, Some(Stop::Candidates)) { + let _ = writeln!( + report, + "backend inventory truncated after {MAX_CANDIDATES} candidates \ + (a further one was found and is not listed)" + ); + } + Inventory { report, servers } +} + +/// The GPU backend directories under `bin/llamacpp` that are searched. The cpu +/// backend is left out on purpose: a CPU build is never what a GPU serve should +/// have launched, and listing it would read as one. +const BACKEND_DIRS: [&str; 4] = ["rocm-stable", "rocm-nightly", "rocm", "vulkan"]; +/// How many directory levels below a backend directory are searched. Lemonade +/// unpacks some llama.cpp builds into a build-numbered directory, and the layout +/// is the router's to choose, so the search is not pinned to one level. +const MAX_SEARCH_DEPTH: usize = 3; +/// Names listed when explaining an empty search. +const MAX_LISTED_NAMES: usize = 16; +/// Directory entries read to build that listing. Bounded so a huge `bin/llamacpp` +/// cannot make the explanation of an empty search expensive; the report says how +/// many names it left out. +const MAX_LISTING_SCAN: usize = 64; + +/// Global bounds on one candidate search. They apply to the whole search, across +/// every backend directory, not per directory: a tree of many shallow directories +/// is as expensive as a deep one. +#[derive(Clone, Copy)] +struct SearchLimits { + /// Directories whose entries are read. + dirs: usize, + /// Directory entries read, in total. + entries: usize, + /// Candidates kept. The search stops as soon as it finds one more than this, + /// which is how it knows the list is cut; the extra one is dropped. + candidates: usize, +} + +const SEARCH_LIMITS: SearchLimits = SearchLimits { + dirs: 128, + entries: 2048, + candidates: MAX_CANDIDATES, +}; + +/// Why a candidate search ended before it had looked everywhere. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stop { + Deadline, + Directories, + Entries, + Candidates, +} + +#[derive(Debug, Default)] +struct Discovery { + /// In a fixed order: backend directories as listed, then sibling directories + /// sorted by path. + candidates: Vec, + stop: Option, + /// Symlinks met and not followed. A cache that links out of its backend + /// directory, or back into it, is neither searched nor allowed to loop the + /// search. + symlinks_skipped: usize, + dirs_visited: usize, + entries_read: usize, +} + +const fn server_binary() -> &'static str { + if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + } +} + +/// Look for `llama-server` under each backend directory, within `limits` and until +/// `expired` says time is up. Symlinks are never followed: only real directories +/// are entered and only a regular file counts as a candidate, so a link cycle +/// ends and a link out of the tree is not read. +fn discover_candidates( + runtime: &Path, + limits: SearchLimits, + expired: &dyn Fn() -> bool, +) -> Discovery { + let mut search = Search { + limits, + expired, + out: Discovery::default(), + }; + for backend in BACKEND_DIRS { + let dir = runtime.join("bin/llamacpp").join(backend); + match std::fs::symlink_metadata(&dir) { + Ok(meta) if meta.is_dir() => search.visit(&dir, 0), + Ok(meta) if meta.file_type().is_symlink() => search.out.symlinks_skipped += 1, + _ => {} + } + if search.out.stop.is_some() { + break; + } + } + search.out.candidates.truncate(limits.candidates); + search.out +} + +struct Search<'a> { + limits: SearchLimits, + expired: &'a dyn Fn() -> bool, + out: Discovery, +} + +impl Search<'_> { + fn stop(&mut self, why: Stop) { + self.out.stop.get_or_insert(why); + } + + fn visit(&mut self, dir: &Path, depth: usize) { + if self.out.stop.is_some() { + return; + } + if (self.expired)() { + self.stop(Stop::Deadline); + return; + } + if self.out.dirs_visited >= self.limits.dirs { + self.stop(Stop::Directories); + return; + } + self.out.dirs_visited += 1; + let direct = dir.join(server_binary()); + // A symlink named llama-server is not a candidate (and is counted below, + // as a directory entry, not here). + if std::fs::symlink_metadata(&direct).is_ok_and(|meta| meta.is_file()) { + self.out.candidates.push(direct); + } + // One more than the cap: finding it is what shows the list is cut, so + // exactly `candidates` of them is a complete list and not a truncated one. + if self.out.candidates.len() > self.limits.candidates { + self.stop(Stop::Candidates); + return; + } + if depth >= MAX_SEARCH_DEPTH { + return; + } + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut subdirs = Vec::new(); + for entry in entries { + if (self.expired)() { + self.stop(Stop::Deadline); + break; + } + if self.out.entries_read >= self.limits.entries { + self.stop(Stop::Entries); + break; + } + self.out.entries_read += 1; + let Ok(entry) = entry else { + continue; + }; + match entry.file_type() { + Ok(kind) if kind.is_dir() => subdirs.push(entry.path()), + Ok(kind) if kind.is_symlink() => self.out.symlinks_skipped += 1, + _ => {} + } + } + subdirs.sort(); + for subdir in subdirs { + self.visit(&subdir, depth + 1); + } + } +} + +/// Why a search found nothing, in terms of where it looked. An absent binary can +/// mean a backend download that did not finish, a layout the search does not +/// cover, or a build that was never installed, and the report cannot tell which, +/// so it says what was searched and what `bin/llamacpp` actually holds. +fn empty_search_note(runtime: &Path) -> String { + let llamacpp = runtime.join("bin/llamacpp"); + let searched = BACKEND_DIRS + .iter() + .map(|backend| llamacpp.join(backend).display().to_string()) + .collect::>() + .join(", "); + let present = match std::fs::read_dir(&llamacpp) { + Ok(entries) => { + let mut names: Vec = entries + .take(MAX_LISTING_SCAN) + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + let hit_scan_cap = names.len() >= MAX_LISTING_SCAN; + names.sort(); + if names.is_empty() { + format!("{} is empty", llamacpp.display()) + } else { + let omitted = names.len().saturating_sub(MAX_LISTED_NAMES); + names.truncate(MAX_LISTED_NAMES); + let more = match (omitted, hit_scan_cap) { + (0, false) => String::new(), + (_, false) => format!(" and {omitted} more"), + (_, true) => format!( + " and at least {omitted} more (only the first {MAX_LISTING_SCAN} \ + entries were read)" + ), + }; + format!( + "{} contains: {}{more}", + llamacpp.display(), + names.join(", ") + ) + } + } + Err(error) => format!("{} cannot be read: {error}", llamacpp.display()), + }; + format!( + "no llama-server candidates found; searched {searched}, down to {MAX_SEARCH_DEPTH} \ + directory levels below each (the cpu backend is not searched); {present}" + ) +} + +/// `limit` as the report words it: whole MiB when it is one, else bytes. +fn describe_hash_limit(limit: u64) -> String { + const MIB: u64 = 1024 * 1024; + if limit.is_multiple_of(MIB) { + format!("{} MiB", limit / MIB) + } else { + format!("{limit} bytes") + } +} + +fn fingerprint(server: &Path, deadline: Instant) -> std::io::Result { + fingerprint_with_limit(server, deadline, MAX_HASH_BYTES) +} + +fn fingerprint_with_limit(server: &Path, deadline: Instant, limit: u64) -> std::io::Result { + let too_large = || { + std::io::Error::other(format!( + "executable exceeds {} hash limit", + describe_hash_limit(limit) + )) + }; + let mut file = open_no_follow(server)?; + // Do not hash an arbitrarily large corrupt cache entry while reporting a + // failure. Never label a partial hash as SHA256. + if file.metadata()?.len() > limit { + return Err(too_large()); + } + let mut hasher = Sha256::new(); + let mut count = 0; + let mut buffer = vec![0; 64 * 1024]; + loop { + // Check inside the worker, not just around its JoinHandle: timing out a + // spawn_blocking await alone would leave the hashing thread running. + if Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "hash evidence budget exhausted", + )); + } + let read = file.read(&mut buffer)?; + count += read as u64; + if count > limit { + return Err(too_large()); + } + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +#[derive(Default)] +struct CapturedOutput { + bytes: Vec, + truncated: bool, +} + +impl CapturedOutput { + fn truncation_note(&self, stream: &str) -> String { + if self.truncated { + format!("\n<{stream} truncated at 32 KiB>") + } else { + String::new() + } + } +} + +async fn probe(server: &Path, flag: &str, timeout: Duration) -> String { + let mut command = tokio::process::Command::new(server); + command.arg(flag); + run_probe(command, timeout).await +} + +/// Run `command` with bounded time and output, and render what came back. +/// +/// Split from [`probe`] so the timeout, kill and reap path can be driven with any +/// child, not only a `llama-server` stand-in: shell-script stand-ins exist on unix +/// only, and this path is what must also hold on Windows. +async fn run_probe(mut command: tokio::process::Command, timeout: Duration) -> String { + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => return format!("probe could not start: {error}"), + }; + let stdout = child.stdout.take().expect("piped stdout"); + let stderr = child.stderr.take().expect("piped stderr"); + let mut out = CapturedOutput::default(); + let mut err = CapturedOutput::default(); + let status = if let Ok((status, stdout_result, stderr_result)) = + tokio::time::timeout(timeout, async { + tokio::join!( + child.wait(), + drain_output(stdout, &mut out), + drain_output(stderr, &mut err), + ) + }) + .await + { + let mut status = status.map_or_else(|error| error.to_string(), |status| status.to_string()); + for (stream, result) in [("stdout", stdout_result), ("stderr", stderr_result)] { + if let Err(error) = result { + let _ = write!(status, "\n{stream} capture failed: {error}"); + } + } + status + } else { + // kill_on_drop also covers cancellation of the whole reporter. + // Explicitly wait here so a normal timeout leaves no zombie child. + let mut status = format!("probe timed out after {}s", timeout.as_secs_f64()); + if let Err(error) = child.kill().await { + let _ = write!(status, "; kill/reap failed: {error}"); + } + status + }; + format!( + "{status}\nstdout:\n{}{}\nstderr:\n{}{}", + String::from_utf8_lossy(&out.bytes), + out.truncation_note("stdout"), + String::from_utf8_lossy(&err.bytes), + err.truncation_note("stderr"), + ) +} + +async fn drain_output( + mut reader: impl AsyncRead + Unpin, + retained: &mut CapturedOutput, +) -> std::io::Result<()> { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + let mut buffer = [0; 4096]; + loop { + let count = reader.read(&mut buffer).await?; + if count == 0 { + return Ok(()); + } + // Continue draining both pipes after the cap, without retaining more + // bytes or writing them to disk. A noisy child must not block on output. + let keep = count.min(limit.saturating_sub(retained.bytes.len())); + retained.bytes.extend_from_slice(&buffer[..keep]); + retained.truncated |= keep < count; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn candidates(runtime: &Path) -> Vec { + discover_candidates(runtime, SEARCH_LIMITS, &|| false).candidates + } + + fn touch_server(dir: &Path) -> PathBuf { + let server = dir.join(server_binary()); + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(&server, "").unwrap(); + server + } + + async fn evidence(manifest: &Path) -> String { + evidence_with(manifest, DEFAULT_BUDGETS).await + } + + async fn evidence_with(manifest: &Path, budgets: Budgets) -> String { + evidence_around_stop(manifest, budgets, &RealProbe, String::new) + .await + .1 + } + + fn write_manifest(runtime: &Path) -> PathBuf { + let manifest = runtime.join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + manifest + } + + /// The candidate paths a report lists, parsed back into paths. + /// + /// Compared as paths, never as printed text: the inventory joins + /// `bin/llamacpp` as one string, so on Windows a report line mixes `/` and + /// `\` while a test that builds the expected path another way prints it + /// differently. `Path` equality compares components, so both spellings agree, + /// and a negative check ("this candidate was skipped") cannot pass vacuously + /// just because the two spellings differ. + fn candidate_paths(report: &str) -> Vec { + report + .lines() + .filter_map(|line| line.strip_prefix("candidate (not proof of selection): ")) + .map(PathBuf::from) + .collect() + } + + #[test] + fn executable_identity_does_not_depend_on_the_dynamic_loader() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("not-executable"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + INVENTORY_BUDGET).unwrap(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn hash_limit_accepts_the_boundary_but_never_returns_a_partial_hash() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + let limit = 3 * 64 * 1024 + 5; + let file = std::fs::File::create(&server).unwrap(); + file.set_len(limit).unwrap(); + assert_eq!( + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) + .unwrap() + .len(), + 64 + ); + file.set_len(limit + 1).unwrap(); + assert_eq!( + fingerprint_with_limit(&server, Instant::now() + INVENTORY_BUDGET, limit) + .unwrap_err() + .to_string(), + format!("executable exceeds {limit} bytes hash limit") + ); + } + + /// The cap is not the old 32 MiB: a server above it, such as a static HIP build, + /// must still get a fingerprint on the lane this report is for. The file is + /// sparse, so the test hashes zeros rather than holding real data. + #[test] + fn a_server_above_the_old_32_mib_cap_is_still_fingerprinted() { + const OLD_CAP: u64 = 32 * 1024 * 1024; + const { assert!(MAX_HASH_BYTES >= 256 * 1024 * 1024) }; + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::File::create(&server) + .unwrap() + .set_len(OLD_CAP + 1) + .unwrap(); + assert_eq!( + fingerprint(&server, Instant::now() + INVENTORY_BUDGET) + .unwrap() + .len(), + 64 + ); + } + + #[test] + fn hash_limit_is_worded_in_the_unit_it_was_set_in() { + assert_eq!(describe_hash_limit(512 * 1024 * 1024), "512 MiB"); + assert_eq!(describe_hash_limit(1000), "1000 bytes"); + } + + #[test] + fn hash_worker_obeys_its_own_deadline() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + std::fs::write(&server, b"abc").unwrap(); + assert_eq!( + fingerprint(&server, Instant::now()).unwrap_err().kind(), + std::io::ErrorKind::TimedOut + ); + } + + #[test] + fn inventory_covers_flat_and_nested_gpu_backends_not_cpu() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let mut wanted = Vec::new(); + for relative in ["rocm-stable", "rocm-nightly/llama-b9752", "vulkan/build"] { + let server = root.path().join("bin/llamacpp").join(relative).join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, "").unwrap(); + wanted.push(server); + } + let cpu = root.path().join("bin/llamacpp/cpu").join(binary); + std::fs::create_dir_all(cpu.parent().unwrap()).unwrap(); + std::fs::write(cpu, "").unwrap(); + assert_eq!(candidates(root.path()), wanted); + } + + #[test] + fn candidate_search_reaches_three_levels_below_a_backend_and_no_further() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-nightly"); + // Literal layouts, not derived from MAX_SEARCH_DEPTH: the depth is the + // behaviour under test, so changing the constant must turn this red. + let found_at_three = backend.join("a/b/llama-b9752").join(server_binary()); + let found_at_two = backend.join("a/build").join(server_binary()); + let beyond = backend.join("a/b/c/llama-b9752").join(server_binary()); + for server in [&found_at_three, &found_at_two, &beyond] { + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + assert_eq!( + candidates(root.path()), + // Sorted by path, so `a/b/...` precedes `a/build/...`. + vec![found_at_three, found_at_two] + ); + } + + /// An empty search says where it looked and what `bin/llamacpp` holds, and does + /// not guess that a download was interrupted: the same report follows from a + /// layout the search does not cover or a build that was never installed. + #[tokio::test] + async fn empty_search_lists_the_searched_locations_and_blames_nothing() { + let root = tempfile::tempdir().unwrap(); + let cpu = root.path().join("bin/llamacpp/cpu"); + std::fs::create_dir_all(&cpu).unwrap(); + std::fs::write(cpu.join(server_binary()), "").unwrap(); + let report = evidence(&write_manifest(root.path())).await; + let llamacpp = root.path().join("bin/llamacpp"); + for backend in BACKEND_DIRS { + assert!( + report.contains(&llamacpp.join(backend).display().to_string()), + "{report}" + ); + } + assert!( + report.contains("no llama-server candidates found"), + "{report}" + ); + assert!( + report.contains(&format!("{} contains: cpu", llamacpp.display())), + "{report}" + ); + assert!( + !report.contains("download") && !report.contains("complete"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + + let bare = tempfile::tempdir().unwrap(); + let report = evidence(&write_manifest(bare.path())).await; + assert!(report.contains("cannot be read"), "{report}"); + } + + fn limits(dirs: usize, entries: usize) -> SearchLimits { + SearchLimits { + dirs, + entries, + candidates: MAX_CANDIDATES, + } + } + + /// The clock is read while the tree is walked, not once before it: it reports + /// time up from its third reading on, so the walk must end partway through, + /// with what it had already found and the reason it stopped. + #[test] + fn the_clock_is_checked_during_the_walk() { + let root = tempfile::tempdir().unwrap(); + let wanted: Vec = BACKEND_DIRS + .iter() + .map(|backend| touch_server(&root.path().join("bin/llamacpp").join(backend))) + .collect(); + let calls = std::sync::atomic::AtomicUsize::new(0); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }); + assert_eq!(found.stop, Some(Stop::Deadline)); + assert_eq!(found.candidates, wanted[..1], "{found:?}"); + let all = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(all.candidates, wanted); + assert_eq!(all.stop, None); + } + + /// The same, through the inventory: a clock that is already up stops the search + /// even though the hashing deadline is far off, so the walk consults the + /// inventory's clock and not only the hash loop's. + #[tokio::test] + async fn an_inventory_whose_time_is_up_skips_the_search() { + let root = tempfile::tempdir().unwrap(); + touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| true), + ) + .await; + let report = &inventory.report; + assert!( + report.contains("remaining evidence skipped: evidence budget exhausted"), + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + assert!( + !report.contains("no llama-server candidates found"), + "an unfinished search must not claim to have found nothing: {report}" + ); + assert!(inventory.servers.is_empty()); + } + + /// The scan is blocking filesystem work and must not run on the async executor. + /// On a current-thread runtime the test's own thread is the executor, so a clock + /// read from any other thread shows the walk ran on a blocking one. + #[tokio::test] + async fn the_directory_scan_runs_off_the_async_executor() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("bin/llamacpp/rocm-stable/build")).unwrap(); + let threads = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let recorded = threads.clone(); + take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(move || { + recorded.lock().unwrap().push(std::thread::current().id()); + false + }), + ) + .await; + let threads = threads.lock().unwrap(); + assert!(!threads.is_empty(), "the walk never read the clock"); + let executor = std::thread::current().id(); + assert!( + threads.iter().all(|thread| *thread != executor), + "the scan ran on the async executor" + ); + } + + /// The limits cover the whole search: two backend directories with three + /// subdirectories each read six entries between them, so a limit of four stops + /// the second directory, which a per-directory limit of four would not. + #[test] + fn entry_and_directory_limits_apply_to_the_whole_search() { + let root = tempfile::tempdir().unwrap(); + for backend in ["rocm-stable", "rocm-nightly"] { + for build in ["a", "b", "c"] { + std::fs::create_dir_all(root.path().join("bin/llamacpp").join(backend).join(build)) + .unwrap(); + } + } + let found = discover_candidates(root.path(), limits(100, 4), &|| false); + assert_eq!(found.stop, Some(Stop::Entries), "{found:?}"); + assert_eq!(found.entries_read, 4, "{found:?}"); + + let found = discover_candidates(root.path(), limits(2, 100), &|| false); + assert_eq!(found.stop, Some(Stop::Directories), "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!( + found.stop, None, + "the default limits are not the small ones" + ); + } + + /// A search that did not finish must not say it found nothing. The tree is the + /// one a reviewer described: an empty `a`, and a real build under `b`, with the + /// cap reached before `b` is entered. Each way a search can stop early reports + /// its own reason and never the completed-empty-search text, which is asserted + /// absent; a search that did finish and found nothing still gets it. (The + /// candidate cap has no row: it stops a search that already has candidates.) + #[tokio::test] + async fn a_search_cut_short_says_why_and_never_claims_an_empty_result() { + const EMPTY_CLAIM: &str = "no llama-server candidates found"; + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(backend.join("a")).unwrap(); + touch_server(&backend.join("b")); + let manifest = write_manifest(root.path()); + let far = Instant::now() + INVENTORY_BUDGET; + let never: Expired = std::sync::Arc::new(|| false); + let always: Expired = std::sync::Arc::new(|| true); + + for (stop, search_limits, clock, stop_line) in [ + ( + Stop::Directories, + limits(2, 100), + never.clone(), + "candidate search stopped: 2 directories visited (limit)", + ), + ( + Stop::Entries, + limits(100, 1), + never.clone(), + "candidate search stopped: 1 directory entries read (limit)", + ), + ( + Stop::Deadline, + SEARCH_LIMITS, + always, + "remaining evidence skipped: evidence budget exhausted", + ), + ] { + let inventory = take_inventory_with(&manifest, far, search_limits, clock).await; + let report = &inventory.report; + assert!(report.contains(stop_line), "{stop:?}: {report}"); + assert!(!report.contains(EMPTY_CLAIM), "{stop:?}: {report}"); + assert!( + !report.contains("candidate (not proof"), + "{stop:?}: {report}" + ); + } + + let finished = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(finished.path().join("bin/llamacpp/rocm-stable/a")).unwrap(); + let inventory = + take_inventory_with(&write_manifest(finished.path()), far, SEARCH_LIMITS, never).await; + assert!( + inventory.report.contains(EMPTY_CLAIM), + "a search that finished and found nothing says so: {}", + inventory.report + ); + } + + /// The candidate cap ends the walk as soon as it is certain the list is cut, + /// instead of the walk finishing and the list being cut afterwards: with twenty + /// candidate directories only the root and the first nine (eight kept, one more + /// to show there are further ones) are entered, in sorted order. + #[test] + fn the_candidate_cap_stops_the_walk_when_it_is_reached() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + let wanted: Vec = (0..20) + .map(|i| touch_server(&backend.join(format!("{i:02}")))) + .collect(); + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, Some(Stop::Candidates)); + assert_eq!(found.candidates, wanted[..MAX_CANDIDATES]); + assert_eq!(found.dirs_visited, 1 + MAX_CANDIDATES + 1, "{found:?}"); + } + + /// Exactly the cap is a complete list, not a truncated one: no stop, and the + /// report prints no truncation note. A ninth candidate makes it truncated, and + /// then the note is printed. Counting the cap itself as "truncated" turns the + /// first half red. + #[tokio::test] + async fn exactly_the_cap_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let backend = root.path().join("bin/llamacpp/rocm-stable"); + for i in 0..MAX_CANDIDATES { + touch_server(&backend.join(format!("{i:02}"))); + } + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.candidates.len(), MAX_CANDIDATES); + let report = evidence(&write_manifest(root.path())).await; + assert!(!report.contains("truncated"), "{report}"); + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + + touch_server(&backend.join(format!("{MAX_CANDIDATES:02}"))); + let report = evidence(&write_manifest(root.path())).await; + assert!( + report.contains("backend inventory truncated after 8 candidates"), + "{report}" + ); + } + + /// When the search ends on the clock after finding candidates, and the hashing + /// deadline has passed too, "remaining evidence skipped" is said once, not once + /// by the search and again by the candidate loop. + #[tokio::test] + async fn the_budget_exhausted_line_is_printed_once() { + let root = tempfile::tempdir().unwrap(); + for backend in BACKEND_DIRS { + touch_server(&root.path().join("bin/llamacpp").join(backend)); + } + let calls = std::sync::atomic::AtomicUsize::new(0); + let calls = std::sync::Arc::new(calls); + let counter = calls.clone(); + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + .checked_sub(Duration::from_secs(1)) + .expect("a second before now"), + SEARCH_LIMITS, + std::sync::Arc::new(move || { + counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2 + }), + ) + .await; + let report = &inventory.report; + assert_eq!( + report + .matches("remaining evidence skipped: evidence budget exhausted") + .count(), + 1, + "{report}" + ); + assert!(!report.contains("candidate (not proof"), "{report}"); + } + + /// A symlink is neither entered nor read as a candidate. The tree holds a link + /// back to its own backend directory (a cycle), a link out to a directory with a + /// real `llama-server`, a `llama-server` that is a link to a file outside, and a + /// whole backend directory that is a link. The walk must end, find only the real + /// file, and count what it declined to follow. + #[cfg(unix)] + #[test] + fn symlinks_are_not_followed_and_a_cycle_ends() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let outside_server = touch_server(outside.path()); + let backends = root.path().join("bin/llamacpp"); + let real = touch_server(&backends.join("rocm-stable/real")); + symlink( + backends.join("rocm-stable"), + backends.join("rocm-stable/loop"), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-stable/out")).unwrap(); + symlink( + &outside_server, + backends.join("rocm-stable").join(server_binary()), + ) + .unwrap(); + symlink(outside.path(), backends.join("rocm-nightly")).unwrap(); + // A version marker beside the real server that links out of the tree. + std::fs::write(outside.path().join("version.txt"), "OUTSIDE-VERSION").unwrap(); + symlink( + outside.path().join("version.txt"), + backends.join("rocm-stable/real/version.txt"), + ) + .unwrap(); + + let found = discover_candidates(root.path(), SEARCH_LIMITS, &|| false); + assert_eq!(found.candidates, vec![real], "{found:?}"); + assert_eq!(found.stop, None, "{found:?}"); + assert_eq!(found.symlinks_skipped, 5, "{found:?}"); + assert_eq!(found.dirs_visited, 2, "{found:?}"); + } + + /// The refusal is made by the open itself, not by a check before it, so a path + /// that becomes a link after any check is still refused. The helper is called + /// directly on a link, which is what a swap after the metadata check looks like + /// to it; removing `O_NOFOLLOW` makes the open succeed and turns this red. A + /// regular file still opens, and a missing one reports its own error, not a + /// refusal. (Unix only: on Windows the check and the open are separate steps.) + #[cfg(unix)] + #[test] + fn the_open_itself_refuses_a_symlink() { + use std::io::Read as _; + use std::os::unix::fs::symlink; + let dir = tempfile::tempdir().unwrap(); + let real = dir.path().join("real"); + std::fs::write(&real, "OUTSIDE").unwrap(); + let link = dir.path().join("link"); + symlink(&real, &link).unwrap(); + + let error = open_no_follow(&link).unwrap_err(); + assert!(is_symlink_refusal(&error), "{error}"); + assert_eq!(error.to_string(), SYMLINK_NOT_FOLLOWED); + + let mut text = String::new(); + open_no_follow(&real) + .unwrap() + .read_to_string(&mut text) + .unwrap(); + assert_eq!(text, "OUTSIDE"); + let missing = open_no_follow(&dir.path().join("missing")).unwrap_err(); + assert!(!is_symlink_refusal(&missing), "{missing}"); + assert_eq!(missing.kind(), std::io::ErrorKind::NotFound); + } + + /// Every file the report reads from the backend tree is refused when it is a + /// symlink, and the report says so, the way the walk reports a link it did not + /// enter. A real `llama-server` sits beside a `version.txt` that links to a file + /// outside the tree, the pins file is a link out too, and so is a manifest: none + /// of the outside content may appear, and each refusal is named. A fingerprint + /// of a path that has become a link is refused as well. + #[cfg(unix)] + #[tokio::test] + async fn linked_files_are_refused_and_named_not_embedded() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + for (name, text) in [ + ("version.txt", "OUTSIDE-VERSION"), + ("pins.json", "OUTSIDE-PINS"), + ("manifest.json", r#"{"version": "OUTSIDE-MANIFEST"}"#), + ("binary", "OUTSIDE-BINARY"), + ] { + std::fs::write(outside.path().join(name), text).unwrap(); + } + let server = touch_server(&root.path().join("bin/llamacpp/rocm-stable")); + std::fs::write(&server, "abc").unwrap(); + symlink( + outside.path().join("version.txt"), + server.parent().unwrap().join("version.txt"), + ) + .unwrap(); + std::fs::create_dir_all(root.path().join("resources")).unwrap(); + symlink( + outside.path().join("pins.json"), + root.path().join("resources/backend_versions.json"), + ) + .unwrap(); + + let inventory = take_inventory_with( + &write_manifest(root.path()), + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + let report = &inventory.report; + assert!(!report.contains("OUTSIDE"), "{report}"); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + server.parent().unwrap().join("version.txt").display() + )), + "{report}" + ); + assert!( + report.contains(&format!( + "{}: {SYMLINK_NOT_FOLLOWED}", + root.path() + .join("resources/backend_versions.json") + .display() + )), + "{report}" + ); + // The real server beside the links is still fingerprinted, and the walk + // counts the one link it saw among the entries. + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!( + report.contains("1 symlinked entries under the backend directories were not followed"), + "{report}" + ); + + let linked_manifest = root.path().join("linked-manifest.json"); + symlink(outside.path().join("manifest.json"), &linked_manifest).unwrap(); + let inventory = take_inventory_with( + &linked_manifest, + Instant::now() + INVENTORY_BUDGET, + SEARCH_LIMITS, + std::sync::Arc::new(|| false), + ) + .await; + assert!( + !inventory.report.contains("OUTSIDE"), + "{}", + inventory.report + ); + assert!( + inventory + .report + .contains(&format!("unavailable: {SYMLINK_NOT_FOLLOWED}")), + "{}", + inventory.report + ); + + let linked_binary = root.path().join("linked-binary"); + symlink(outside.path().join("binary"), &linked_binary).unwrap(); + assert_eq!( + fingerprint(&linked_binary, Instant::now() + INVENTORY_BUDGET) + .unwrap_err() + .to_string(), + SYMLINK_NOT_FOLLOWED + ); + } + + #[tokio::test] + async fn the_empty_search_listing_is_bounded_and_counts_what_it_omits() { + let root = tempfile::tempdir().unwrap(); + let llamacpp = root.path().join("bin/llamacpp"); + for i in 0..(MAX_LISTED_NAMES + 4) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + assert!(report.contains(" and 4 more"), "{report}"); + assert!(!report.contains("at least"), "{report}"); + + for i in 0..(2 * MAX_LISTING_SCAN) { + std::fs::create_dir_all(llamacpp.join(format!("other-{i:03}"))).unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + let listing = report + .split("contains: ") + .nth(1) + .and_then(|rest| rest.lines().next()) + .unwrap(); + let (names, omitted) = listing.split_once(" and at least ").unwrap(); + assert_eq!(names.split(", ").count(), MAX_LISTED_NAMES, "{report}"); + assert!( + omitted.starts_with(&format!("{} more", MAX_LISTING_SCAN - MAX_LISTED_NAMES)), + "{report}" + ); + assert!( + omitted.contains(&format!("first {MAX_LISTING_SCAN} entries")), + "{report}" + ); + } + + #[tokio::test] + async fn evidence_truncates_candidates_without_hashing_or_probing_the_rest() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + for i in 0..=MAX_CANDIDATES { + let server = root + .path() + .join(format!("bin/llamacpp/rocm-stable/{i:02}/{binary}")); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + } + let report = evidence(&write_manifest(root.path())).await; + assert_eq!( + report + .matches("candidate (not proof of selection):") + .count(), + MAX_CANDIDATES, + "{report}" + ); + assert_eq!(report.matches("sha256:").count(), MAX_CANDIDATES); + assert_eq!(report.matches("--help:").count(), MAX_CANDIDATES); + assert!(report.contains("backend inventory truncated after 8 candidates")); + let skipped = root + .path() + .join(format!("bin/llamacpp/rocm-stable/08/{binary}")); + let listed = candidate_paths(&report); + assert_eq!(listed.len(), MAX_CANDIDATES, "{report}"); + assert!(!listed.contains(&skipped), "{report}"); + let first = root + .path() + .join(format!("bin/llamacpp/rocm-stable/00/{binary}")); + assert!( + listed.contains(&first), + "the comparison must be able to match a listed candidate: {report}" + ); + } + + /// An executable past the hash cap is not fingerprinted, but the report says + /// so instead of omitting the line, and still lists the candidate and probes it. + #[tokio::test] + async fn oversized_executable_reports_why_it_has_no_fingerprint() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::File::create(&server) + .unwrap() + .set_len(MAX_HASH_BYTES + 1) + .unwrap(); + let report = evidence(&write_manifest(root.path())).await; + assert!( + report.contains(&format!( + "sha256 unavailable: executable exceeds {} hash limit", + describe_hash_limit(MAX_HASH_BYTES) + )), + "{report}" + ); + assert!(!report.contains("sha256: "), "{report}"); + assert_eq!(candidate_paths(&report), vec![server], "{report}"); + assert!(report.contains("--version:"), "{report}"); + } + + #[tokio::test] + async fn exhausted_budget_skips_hashing_and_probes() { + let root = tempfile::tempdir().unwrap(); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = root.path().join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(server, "").unwrap(); + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: Duration::ZERO, + probes: PROBE_BUDGET, + }, + ) + .await; + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + for skipped in ["sha256", "--version:", "--help:", "candidate ("] { + assert!(!report.contains(skipped), "{report}"); + } + } + + type Events = std::sync::Mutex>; + + struct RecordingProbe<'a> { + events: &'a Events, + } + + impl Probe for RecordingProbe<'_> { + async fn run(&self, _server: &Path, flag: &str, _timeout: Duration) -> String { + self.events.lock().unwrap().push(format!("probe {flag}")); + "recorded, not executed".to_owned() + } + } + + /// A lemonade data dir whose manifest sits where the engine writes it, with one + /// small candidate whose SHA-256 is a known value. + fn lemonade_data_dir_with_candidate() -> (tempfile::TempDir, PathBuf, PathBuf) { + let root = tempfile::tempdir().unwrap(); + let paths = rocm_core::AppPaths { + config_dir: root.path().join("config"), + data_dir: root.path().join("data"), + cache_dir: root.path().join("cache"), + }; + let runtime = root.path().join("runtime"); + let binary = if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }; + let server = runtime.join("bin/llamacpp/rocm-stable").join(binary); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + std::fs::write(&server, b"abc").unwrap(); + let manifest = paths.engine_install_manifest_path("lemonade"); + std::fs::create_dir_all(manifest.parent().unwrap()).unwrap(); + std::fs::write( + &manifest, + serde_json::json!({"runtime_dir": runtime}).to_string(), + ) + .unwrap(); + (root, manifest, server) + } + + const ABC_SHA256: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + + /// The inventory is read while the failed service is still in place and the + /// probes, which execute a backend binary, run only after the stop. + /// + /// The stop deletes the candidate. A fingerprint in the report therefore proves + /// the file was read before the stop, and the recorded events prove the probes + /// came after it; running either step on the wrong side turns this red. + #[tokio::test] + async fn inventory_precedes_the_stop_and_probes_follow_it() { + let (_root, manifest, server) = lemonade_data_dir_with_candidate(); + let events = Events::default(); + let (stop_status, report) = evidence_around_stop( + &manifest, + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || { + events.lock().unwrap().push("stop".to_owned()); + std::fs::remove_file(&server).unwrap(); + "stopped".to_owned() + }, + ) + .await; + assert_eq!(stop_status, "stopped"); + assert_eq!( + *events.lock().unwrap(), + ["stop", "probe --version", "probe --help"], + "{report}" + ); + assert!( + report.contains(&format!("sha256: {ABC_SHA256}")), + "{report}" + ); + assert!(!report.contains("sha256 unavailable"), "{report}"); + let inventory = report.find("candidate (not proof of selection):").unwrap(); + let probes = report.find("probe target:").unwrap(); + assert!(inventory < probes, "{report}"); + assert!(report.contains("recorded, not executed"), "{report}"); + } + + #[tokio::test] + async fn stop_runs_once_for_every_engine_and_an_unknown_engine_stays_visible() { + let stops = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "stopped".to_owned() + }; + let no_manifest = + || -> Result { panic!("the manifest is only read for lemonade") }; + + assert_eq!( + backend_evidence_around_stop(no_manifest, Some("vllm"), stop).await, + ("stopped".to_owned(), None) + ); + assert_eq!( + backend_evidence_around_stop(no_manifest, None, stop).await, + ( + "stopped".to_owned(), + Some(ENGINE_UNKNOWN_EVIDENCE.to_owned()) + ) + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); + + let (_root, manifest, _server) = lemonade_data_dir_with_candidate(); + let (status, evidence) = + backend_evidence_around_stop(|| Ok(manifest), Some("lemonade"), stop).await; + assert_eq!(status, "stopped"); + assert!(evidence.unwrap().contains("manifest: ")); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 3); + } + + /// A Lemonade serve with nothing to look the manifest up in (the scenario has no + /// isolated root) used to panic before the stop, so the stop never ran and the + /// whole failure report was lost. The lookup now fails soft, whether it says so + /// (`Err`) or panics, and the stop still runs exactly once. Removing the panic + /// guard turns the second case red. + #[tokio::test] + async fn a_failing_manifest_lookup_still_runs_the_stop_once_and_reports_why() { + let stops = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + "stopped".to_owned() + }; + let (status, evidence) = backend_evidence_around_stop( + || Err("the scenario has no isolated root".to_owned()), + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: the scenario has no isolated root") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 1); + + let (status, evidence) = backend_evidence_around_stop( + || -> Result { panic!("no isolated root") }, + Some("lemonade"), + stop, + ) + .await; + assert_eq!(status, "stopped"); + assert_eq!( + evidence.as_deref(), + Some("backend evidence unavailable: finding the runtime manifest failed") + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn an_unreadable_manifest_is_reported_and_the_service_is_still_stopped() { + let root = tempfile::tempdir().unwrap(); + let events = Events::default(); + let (stop_status, report) = evidence_around_stop( + &root.path().join("missing.json"), + DEFAULT_BUDGETS, + &RecordingProbe { events: &events }, + || "stopped".to_owned(), + ) + .await; + assert_eq!(stop_status, "stopped"); + assert!(report.contains("unavailable:"), "{report}"); + assert!(events.lock().unwrap().is_empty(), "no candidate, no probe"); + assert!(!report.contains("probe target:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn timed_out_probe_leaves_no_budget_for_the_next_probe() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("bin/llamacpp/rocm-stable/llama-server"); + std::fs::create_dir_all(server.parent().unwrap()).unwrap(); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nexec /bin/sleep 30\n", + ) + .unwrap(); + let report = evidence_with( + &write_manifest(root.path()), + Budgets { + inventory: INVENTORY_BUDGET, + probes: Duration::from_secs(1), + }, + ) + .await; + assert!(report.contains("--version:\nprobe timed out"), "{report}"); + assert!(report.contains("remaining evidence skipped: evidence budget exhausted")); + assert!(!report.contains("--help:"), "{report}"); + } + + #[tokio::test] + async fn stream_capture_marks_only_bytes_actually_discarded() { + let limit = MAX_EVIDENCE_BYTES as usize / 2; + for length in [0, limit - 1, limit, limit + 1] { + let bytes = vec![b'x'; length]; + let mut captured = CapturedOutput::default(); + drain_output(bytes.as_slice(), &mut captured).await.unwrap(); + assert_eq!(captured.bytes, vec![b'x'; length.min(limit)]); + assert_eq!(captured.truncated, length > limit); + assert_eq!( + captured.truncation_note("stdout"), + if length > limit { + "\n" + } else { + "" + } + ); + } + } + + #[tokio::test] + async fn stream_truncation_survives_timeout_before_eof() { + use tokio::io::AsyncWriteExt as _; + let length = MAX_EVIDENCE_BYTES as usize / 2 + 1; + let (reader, mut writer) = tokio::io::duplex(length); + writer.write_all(&vec![b'x'; length]).await.unwrap(); + let mut captured = CapturedOutput::default(); + assert!( + tokio::time::timeout( + Duration::from_millis(50), + drain_output(reader, &mut captured) + ) + .await + .is_err() + ); + assert_eq!(captured.bytes.len(), length - 1); + assert_eq!( + captured.truncation_note("stderr"), + "\n" + ); + } + + #[tokio::test] + async fn missing_manifest_is_reported_without_panicking() { + let root = tempfile::tempdir().unwrap(); + let report = evidence(&root.path().join("missing.json")).await; + assert!(report.contains("unavailable:"), "{report}"); + } + + #[tokio::test] + async fn failed_download_keeps_pins_but_does_not_invent_an_installed_build() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + std::fs::create_dir_all(runtime.join("resources")).unwrap(); + std::fs::write( + runtime.join("resources/backend_versions.json"), + r#"{"llamacpp":{"rocm-stable":"b9752"}}"#, + ) + .unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = evidence(&manifest).await; + assert!( + report.contains("configured pins (not executable identity):"), + "{report}" + ); + assert!(report.contains("\"rocm-stable\":\"b9752\""), "{report}"); + assert!(report.contains("no llama-server candidates"), "{report}"); + } + + #[tokio::test] + async fn probe_start_error_is_evidence_not_a_second_failure() { + let root = tempfile::tempdir().unwrap(); + let report = probe(&root.path().join("missing"), "--version", PROBE_TIMEOUT).await; + assert!(report.contains("probe could not start:"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn inventory_probes_the_cached_executable_not_a_path_lookup() { + let root = tempfile::tempdir().unwrap(); + let runtime = root.path().join("runtime"); + let dir = runtime.join("bin/llamacpp/rocm-stable"); + std::fs::create_dir_all(&dir).unwrap(); + let server = dir.join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "\ +#!/bin/sh +case \"$1\" in + --version) echo 'version: 9752 (fixture-build)' ;; + --help) echo '--mmap, --no-mmap'; echo 'fixture help stderr' >&2; exit 2 ;; + *) exit 99 ;; +esac +", + ) + .unwrap(); + std::fs::write(dir.join("version.txt"), "cached-build").unwrap(); + let manifest = root.path().join("runtime.json"); + std::fs::write( + &manifest, + serde_json::json!({ + "version": "fixture", + "runtime_dir": runtime, + "backend_name": "rocm" + }) + .to_string(), + ) + .unwrap(); + let report = evidence(&manifest).await; + assert!(report.contains(&server.display().to_string()), "{report}"); + assert!(report.contains("version: 9752 (fixture-build)"), "{report}"); + assert!(report.contains("version.txt: cached-build"), "{report}"); + assert!( + report.contains("--help:\nexit status: 2\nstdout:"), + "{report}" + ); + assert!(!report.contains("capture"), "{report}"); + assert!(!report.contains("Ok("), "{report}"); + assert!(report.contains("--mmap, --no-mmap"), "{report}"); + assert!(report.contains("fixture help stderr"), "{report}"); + assert!( + server.is_file(), + "diagnostics must not repair or remove the cache" + ); + } + + /// Selects what [`probe_child`] does when a probe test re-runs this test binary + /// as its child process. + const PROBE_CHILD_ENV: &str = "ROCM_E2E_PROBE_CHILD"; + + /// Not a test: the child process the probe tests below start. + /// + /// The probe's timeout, kill and reap path has to hold on Windows too, and a + /// shell-script stand-in does not exist there. This test binary does, on every + /// platform, so the tests re-run it with `--ignored --exact` and this function + /// plays the stand-in. It is `#[ignore]`d so a normal run skips it, and does + /// nothing unless [`PROBE_CHILD_ENV`] selects a behaviour. + #[test] + #[ignore = "child process of the probe tests, not a test of its own"] + fn probe_child() { + match std::env::var(PROBE_CHILD_ENV).as_deref() { + Ok("sleep") => { + // With one test thread (RUST_TEST_THREADS=1, or one CPU) libtest + // writes "test ... " first and our output continues that + // line, so start a fresh one: the parent reads the first line that + // begins with the marker. + println!("\npid:{}", std::process::id()); + std::thread::sleep(Duration::from_secs(CHILD_LIFETIME_SECS)); + } + Ok("exit") => println!("child-ran"), + _ => {} + } + } + + fn probe_child_command(behaviour: &str) -> tokio::process::Command { + let module = module_path!().split_once("::").unwrap().1; + let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + &format!("{module}::probe_child"), + "--ignored", + "--nocapture", + ]) + .env(PROBE_CHILD_ENV, behaviour); + command + } + + /// How long each attempt at the timeout test lets the helper child run before the + /// probe's timeout fires. The first is short, so a healthy run is quick; the + /// later ones are generous and matter only when the child was too slow to start + /// and print its pid inside the earlier one (a loaded runner, a cold test + /// binary), which is a slow start and not a failure of the probe. + const CHILD_START_ATTEMPTS: [u64; 3] = [5, 20, 60]; + + /// How much longer than the longest attempt the helper child stays alive. The + /// child must still be running when the probe's timeout fires on any attempt: + /// one that had exited by itself would be a normal exit, not a timeout, and the + /// test would fail for a correct probe. + const CHILD_LIFETIME_MARGIN_SECS: u64 = 60; + + /// The helper child's lifetime, derived from the attempt timeouts so the two + /// cannot drift apart. + const CHILD_LIFETIME_SECS: u64 = { + let mut longest = 0; + let mut at = 0; + while at < CHILD_START_ATTEMPTS.len() { + if CHILD_START_ATTEMPTS[at] > longest { + longest = CHILD_START_ATTEMPTS[at]; + } + at += 1; + } + longest + CHILD_LIFETIME_MARGIN_SECS + }; + + /// The helper child outlives every attempt's timeout by at least half a minute + /// (the margin is asserted as a literal so shrinking the constant fails), so the + /// timeout, kill and reap path is what ends it on each retry. Giving the child a + /// literal lifetime that a longer attempt can overtake (it was 30 seconds against + /// a 60-second attempt) turns this red. + #[test] + fn the_helper_child_outlives_every_attempt_deadline() { + for attempt in CHILD_START_ATTEMPTS { + assert!( + CHILD_LIFETIME_SECS >= attempt + 30, + "the child lives {CHILD_LIFETIME_SECS}s but an attempt waits {attempt}s" + ); + } + } + + /// One run of the timeout path against a helper child that libtest runs with + /// `test_threads` test threads: the child is still running when the timeout + /// fires, so the report must say so, return promptly, and leave no process + /// behind. `process_is_running` is false only once the child has been killed + /// and reaped, so a probe that merely abandoned it turns this red. + /// + /// The pid is read back from the report, so an attempt whose child had not + /// started before the timeout (no pid line) is retried with a longer timeout + /// rather than failed; only an attempt that did report a pid is asserted on. + async fn assert_timeout_kills_and_reaps(test_threads: &str) { + let mut last = String::new(); + for seconds in CHILD_START_ATTEMPTS { + let start = Instant::now(); + let mut command = probe_child_command("sleep"); + command.env("RUST_TEST_THREADS", test_threads); + let report = run_probe(command, Duration::from_secs(seconds)).await; + assert!( + report.starts_with(&format!("probe timed out after {seconds}s")), + "{report}" + ); + assert!(!report.contains("kill/reap failed"), "{report}"); + assert!( + start.elapsed() < Duration::from_secs(seconds + 15), + "{report}" + ); + let Some(pid) = report + .lines() + .find_map(|line| line.strip_prefix("pid:")) + .map(|pid| pid.parse::().unwrap()) + else { + last = report; + continue; + }; + assert!(!rocm_core::process_is_running(pid), "{report}"); + return; + } + panic!( + "the child never reported its pid, even with {CHILD_START_ATTEMPTS:?} seconds \ + (test threads {test_threads}): {last}" + ); + } + + /// The timeout path on every platform, with the child run both on one test + /// thread and on several. With one thread (also what a one-CPU host gets) libtest + /// prefixes the child's output with its test-name label, so a pid printed on + /// that same line is not found by a line-start match. The two runs overlap, so + /// this takes one timeout, not two. + #[tokio::test] + async fn a_probe_that_outlives_its_timeout_is_killed_and_reaped() { + tokio::join!( + assert_timeout_kills_and_reaps("1"), + assert_timeout_kills_and_reaps("8"), + ); + } + + /// The normal path on every platform: status and output come back, and the + /// timeout branch is not taken. + #[tokio::test] + async fn a_probe_that_exits_reports_its_status_and_output() { + let report = run_probe(probe_child_command("exit"), Duration::from_mins(1)).await; + let status = if cfg!(windows) { + "exit code: 0\n" + } else { + "exit status: 0\n" + }; + assert!(report.starts_with(status), "{report}"); + assert!(report.contains("child-ran"), "{report}"); + assert!(!report.contains("timed out"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn probe_timeout_is_bounded() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + // exec keeps the sleeper in the child's PID, so cancellation kills it + // rather than leaving a grandchild behind. + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\necho $$\nexec /bin/sleep 30\n", + ) + .unwrap(); + let start = std::time::Instant::now(); + let report = probe(&server, "--help", Duration::from_secs(1)).await; + assert!( + report.contains("probe timed out after 1s\nstdout:"), + "{report}" + ); + assert!(start.elapsed() < Duration::from_secs(5)); + let pid = report + .split("stdout:\n") + .nth(1) + .unwrap() + .lines() + .next() + .unwrap() + .parse::() + .unwrap(); + assert!(!rocm_core::process_is_running(pid), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn exact_stream_limit_is_not_reported_as_truncated() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 32768 /dev/zero\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains("exit status: 0"), "{report}"); + assert_eq!(report.bytes().filter(|byte| *byte == 0).count(), 32768); + assert!(!report.contains("capped"), "{report}"); + assert!(!report.contains("truncated"), "{report}"); + } + + #[cfg(unix)] + #[tokio::test] + async fn noisy_probe_output_is_capped() { + let root = tempfile::tempdir().unwrap(); + let server = root.path().join("llama-server"); + rocm_core::fake_executable::write_fake_executable( + &server, + "#!/bin/sh\nhead -c 100000 /dev/zero\nhead -c 100000 /dev/zero >&2\n", + ) + .unwrap(); + let report = probe(&server, "--help", PROBE_TIMEOUT).await; + assert!(report.contains(""), "{report}"); + assert!(report.ends_with(""), "{report}"); + assert_eq!( + report.bytes().filter(|byte| *byte == 0).count(), + MAX_EVIDENCE_BYTES as usize + ); + assert!(report.len() < MAX_EVIDENCE_BYTES as usize + 300); + } +} diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index acd458f969..520ceb1574 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -9,6 +9,7 @@ pub mod fixture_binary; pub mod harness_diagnostics; pub mod http_server; pub mod installer_fixture; +pub mod lemonade_diagnostics; pub mod loopback_http; pub mod mock_server; pub mod model_id; diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index bf0ff9d8c1..90d7573ecc 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -55,6 +55,44 @@ fn parse_log_path(serve_stdout: &str) -> Option<&str> { .filter(|path| !path.is_empty()) } +/// The selected engine, if the serve reached the point of printing its plan. +#[must_use] +pub fn reported_engine(serve_stdout: &str) -> Option<&str> { + serve_stdout + .lines() + .find_map(|line| line.trim().strip_prefix("engine:")) + .map(str::trim) + .filter(|engine| !engine.is_empty()) +} + +/// The engine named on a `rocm serve` command line. +/// +/// Accepts `--engine X` and `--engine=X`. `None` when no engine was requested: +/// the CLI then picks a default, which only the printed plan reveals (see +/// [`reported_engine`]). A value that starts with `--` is another flag, not an +/// engine (`serve m --engine --managed`), so it counts as absent too: reporting +/// `--managed` as the engine would skip the backend inventory without the +/// [`ENGINE_UNKNOWN_EVIDENCE`] line that makes the skip visible. +#[must_use] +pub fn engine_from_serve_args<'a>(args: &[&'a str]) -> Option<&'a str> { + let mut args = args.iter().copied(); + while let Some(arg) = args.next() { + let value = if let Some(inline) = arg.strip_prefix("--engine=") { + Some(inline) + } else if arg == "--engine" { + args.next() + } else { + continue; + }; + return value.filter(|engine| !engine.is_empty() && !engine.starts_with("--")); + } + None +} + +/// What a failed serve's report says when no engine could be determined, so a +/// skipped backend inventory is visible rather than silently absent. +pub const ENGINE_UNKNOWN_EVIDENCE: &str = "engine unknown: backend evidence skipped"; + /// The last `max` lines of `text`. fn tail_lines(text: &str, max: usize) -> String { let lines: Vec<&str> = text.lines().collect(); @@ -119,16 +157,55 @@ fn slugify(name: &str) -> String { /// this is collecting. Seeking keeps the cost fixed however big the log grows. /// /// The log may still be being written, so `len` is a snapshot: the head and tail -/// are always real, and only the elided count can be slightly stale. -fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { +/// are always real, and only the elided count can be slightly stale. A log that +/// grows past the cap after the size was taken is clamped the same way and the +/// note says it grew (see [`read_clamped_file`]); no caller gets a silent +/// truncation. Callers: `archive_service_log` (the note lands on the archived-path +/// line) and the Lemonade report's pins and `version.txt` reads (the note follows +/// the quoted text); its manifest read discards the note, but a truncated JSON +/// manifest fails to parse and the report says so. +pub(crate) fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { + let file = std::fs::File::open(path)?; + let len = file.metadata()?.len(); + read_clamped_file(file, len, max) +} + +/// [`read_clamped`] on an open file whose size was read as `len`. Split out so a +/// test can hand it a `len` that has gone stale, which is what a log that grows +/// between the size check and the read looks like. +/// +/// A file that fits is read in full, but one byte past `max` is asked for: if it +/// is there, the log grew after `len` was taken. That is not allowed to pass as +/// "fits" with the surplus dropped without a word, so the read falls back to the +/// head and tail of the file as it is now and the note says it grew. A file that +/// shrank instead is returned as read, with a note that it changed. +pub(crate) fn read_clamped_file( + mut file: std::fs::File, + len: u64, + max: u64, +) -> std::io::Result<(Vec, String)> { use std::io::{Read as _, Seek as _, SeekFrom}; - let mut file = std::fs::File::open(path)?; - let len = file.metadata()?.len(); + let mut len = len; + let mut grown = false; if len <= max { let mut body = Vec::new(); - file.read_to_end(&mut body)?; - return Ok((body, String::new())); + file.by_ref() + .take(max.saturating_add(1)) + .read_to_end(&mut body)?; + if body.len() as u64 <= max { + return Ok((body, String::new())); + } + len = file.metadata()?.len(); + if len <= max { + body.truncate(usize::try_from(max).unwrap_or(usize::MAX)); + return Ok(( + body, + format!(" (truncated at {max} bytes: the file changed while it was read)"), + )); + } + file.seek(SeekFrom::Start(0))?; + grown = true; } // `len > max >= 2 * half`, so the two halves can never overlap. let half = max / 2; @@ -143,7 +220,15 @@ fn read_clamped(path: &Path, max: u64) -> std::io::Result<(Vec, String)> { let mut body = head; body.extend_from_slice(marker.as_bytes()); body.extend_from_slice(&tail); - Ok((body, format!(" ({elided} bytes elided from the middle)"))) + let grown_note = if grown { + "; the file grew while it was read" + } else { + "" + }; + Ok(( + body, + format!(" ({elided} bytes elided from the middle{grown_note})"), + )) } /// Copy the log written by the managed service `serve_stdout` launched into @@ -230,6 +315,8 @@ pub struct ServeAttempt<'a> { /// failed, means the engine may still hold the port and the device — which /// changes how every later failure in the run should be read. pub stop_status: &'a str, + /// Only Lemonade attempts carry the optional backend inventory. + pub backend_evidence: Option<&'a str>, } /// Render one serve attempt's evidence as a failure message. @@ -241,7 +328,7 @@ pub struct ServeAttempt<'a> { #[must_use] pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { use crate::section; - format!( + let mut report = format!( "{}\n{}\n{}\n{}\n{}\n{}\n{}", attempt.headline, section("device state", attempt.device_state), @@ -250,14 +337,20 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { section("service log (tail)", attempt.log_tail), section("service log (full)", attempt.archived_log), section("stop", attempt.stop_status), - ) + ); + if let Some(evidence) = attempt.backend_evidence { + report.push_str("\n--- Lemonade backend evidence ---\n"); + report.push_str(evidence); + } + report } #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, parse_log_path, read_clamped, - serve_attempt_report, service_log_tail, slugify, tail_lines, + MAX_ARCHIVED_LOG_BYTES, ServeAttempt, archive_service_log, engine_from_serve_args, + parse_log_path, read_clamped, read_clamped_file, reported_engine, serve_attempt_report, + service_log_tail, slugify, tail_lines, }; const SERVE_PLAN: &str = "\ @@ -270,6 +363,38 @@ managed service launched readiness: starting "; + #[test] + fn engine_lookup_is_safe_for_missing_or_incomplete_serve_plans() { + assert_eq!(reported_engine(SERVE_PLAN), Some("vllm")); + assert_eq!(reported_engine(" engine: lemonade\n"), Some("lemonade")); + for output in ["", "serve failed", " engine: \n"] { + assert_eq!(reported_engine(output), None); + } + } + + #[test] + fn engine_is_read_from_both_flag_forms_and_absent_otherwise() { + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine", "lemonade", "--managed"]), + Some("lemonade") + ); + assert_eq!( + engine_from_serve_args(&["serve", "m", "--engine=lemonade", "--managed"]), + Some("lemonade") + ); + for args in [ + &["serve", "m", "--managed"][..], + &["serve", "m", "--engine"], + &["serve", "m", "--engine="], + &["serve", "m", "--engine", ""], + &["serve", "m", "--engine", "--managed"], + &["serve", "m", "--engine=--managed"], + &["serve", "--engine-version=1"], + ] { + assert_eq!(engine_from_serve_args(args), None, "{args:?}"); + } + } + #[test] fn log_path_is_read_from_the_indented_plan_line() { assert_eq!( @@ -395,6 +520,34 @@ managed service launched read_clamped(&path, max).expect("read log") } + /// A log that grows after its size was read must not be cut at the cap without + /// saying so. `len` is handed in stale (10 bytes under a 20-byte cap) for a file + /// that now holds 60: the read keeps the head and tail of what is there and the + /// note says it grew. Cutting at the cap silently turns this red, and a file + /// that really fits (stale or not) stays verbatim with no note. + #[test] + fn a_log_that_grew_after_its_size_was_read_is_clamped_and_says_so() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("service.log"); + let body: Vec = (0..60u8).map(|i| b'a' + i % 26).collect(); + std::fs::write(&path, &body).expect("write log"); + + let (clamped, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 10, 20).expect("read"); + assert!(note.contains("the file grew while it was read"), "{note}"); + assert!(note.contains("40 bytes elided from the middle"), "{note}"); + let text = String::from_utf8_lossy(&clamped); + assert!(text.starts_with(&String::from_utf8_lossy(&body[..10]).to_string())); + assert!(text.ends_with(&String::from_utf8_lossy(&body[50..]).to_string())); + + let (whole, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 100).expect("read"); + assert_eq!((whole, note), (body.clone(), String::new())); + let (exact, note) = + read_clamped_file(std::fs::File::open(&path).expect("open"), 60, 60).expect("read"); + assert_eq!((exact, note), (body, String::new())); + } + /// A runaway log is bounded, but never at the cost of the startup banner: /// both ends survive and the gap says so. #[test] @@ -502,7 +655,7 @@ managed service launched /// full log went — none of which the plain readiness timeout reported. #[test] fn a_stalled_serve_report_carries_every_piece_of_evidence() { - let report = serve_attempt_report(&ServeAttempt { + let mut attempt = ServeAttempt { headline: "`rocm serve unsloth/Qwen3-0.6B-GGUF:Q4_0 --engine lemonade --managed` \ exited 0, but the endpoint never served `Qwen3-0.6B` within 600s", device_state: "device state: drained (48000 MiB free of 49000 MiB, floor 44100 MiB)", @@ -511,7 +664,15 @@ managed service launched log_tail: "load: failed to load model", archived_log: "archived to service-logs/scenario--svc.log", stop_status: "lemonade-qwen: stopped", - }); + backend_evidence: None, + }; + let report = serve_attempt_report(&attempt); + assert!(!report.contains("Lemonade backend evidence"), "{report}"); + attempt.backend_evidence = Some("fixture backend identity"); + assert_eq!( + serve_attempt_report(&attempt), + format!("{report}\n--- Lemonade backend evidence ---\nfixture backend identity") + ); for expected in [ "exited 0", diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 5e126d1f01..d3e236b973 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -13,7 +13,8 @@ use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, + ServeAttempt, archive_service_log, engine_from_serve_args, reported_engine, + serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -136,11 +137,15 @@ fn stall_headline(invocation: &str, ready_substr: &str, timeout_secs: u64) -> St /// rather than three calls at each site: the log must be read and copied out /// while the service still owns it, because stopping the service both changes /// what the log's last lines say and is the step most likely to fail outright. +/// For Lemonade the backend inventory is read before the stop too, and the +/// `llama-server` probes, which execute a backend binary, run only after it so +/// they cannot contend with the still-running router. /// /// Mirrors what [`setup_gpu_model`] has always collected, so a stalled serve /// reports the same evidence wherever it happens. -fn serve_failure_evidence( - world: &E2eWorld, +async fn serve_failure_evidence( + world: &mut E2eWorld, + engine: Option<&str>, headline: &str, device_state: &str, stdout: &str, @@ -155,7 +160,24 @@ fn serve_failure_evidence( &crate::results_path(), world.current_scenario.as_deref().unwrap_or("scenario"), ); - let stop_status = stop_scenario_services(world); + // The stop sits inside this call because its position relative to the + // backend probes is the point: see `backend_evidence_around_stop`. + let app_paths = world.isolated_root.is_some().then(|| world.app_paths()); + let isolated_root = world + .isolated_root + .as_ref() + .map(|root| root.path().to_owned()); + let (stop_status, backend_evidence) = + e2e_cucumber::lemonade_diagnostics::backend_evidence_around_stop( + move || { + app_paths + .map(|paths| paths.engine_install_manifest_path("lemonade")) + .ok_or_else(|| "the scenario has no isolated root".to_owned()) + }, + engine, + move || stop_services_in(isolated_root.as_deref()), + ) + .await; serve_attempt_report(&ServeAttempt { headline, device_state, @@ -164,6 +186,7 @@ fn serve_failure_evidence( log_tail: &log_tail, archived_log: &archived_log, stop_status: &stop_status, + backend_evidence: backend_evidence.as_deref(), }) } @@ -202,7 +225,15 @@ async fn serve_and_wait( }; panic!( "{}", - serve_failure_evidence(world, &headline, &device_state, &stdout, &stderr) + serve_failure_evidence( + world, + engine_from_serve_args(args).or_else(|| reported_engine(&stdout)), + &headline, + &device_state, + &stdout, + &stderr, + ) + .await ); } @@ -543,9 +574,16 @@ fn default_engine_serve_target() -> &'static str { /// attempt shares the device with this one, and that must be visible in the /// failure rather than inferred from a serve that looks broken. fn stop_scenario_services(world: &E2eWorld) -> String { - world.isolated_root.as_ref().map_or_else( + stop_services_in(world.isolated_root.as_ref().map(tempfile::TempDir::path)) +} + +/// [`stop_scenario_services`] on the scenario's isolated root rather than the +/// world, so a caller that must hand the stop to a `Send` closure (the world holds +/// PTY handles and is not `Sync`) can capture the path alone. +fn stop_services_in(isolated_root: Option<&std::path::Path>) -> String { + isolated_root.map_or_else( || "not attempted: scenario has no isolated root".to_owned(), - |root| crate::stop_managed_services(root.path()), + crate::stop_managed_services, ) } @@ -658,13 +696,17 @@ async fn serve_gpu_model(world: &mut E2eWorld) { // outcome is quoted: a stop that found no record (the launch had not yet // written one) or failed means the next attempt did NOT get a clean // device, and the failure says so instead of looking like a broken serve. - diagnostics.push(serve_failure_evidence( - world, - &format!("attempt {attempt} of {attempts} (rc={rc})"), - &device_state, - &stdout, - &stderr, - )); + diagnostics.push( + serve_failure_evidence( + world, + Some(engine), + &format!("attempt {attempt} of {attempts} (rc={rc})"), + &device_state, + &stdout, + &stderr, + ) + .await, + ); if attempt == attempts { break; } @@ -1041,15 +1083,19 @@ async fn user_serves_default_engine(world: &mut E2eWorld) { &ready_substr, timeout_secs, ); + let stdout = world.cli_output.clone().unwrap_or_default(); + let stderr = world.cli_stderr.clone().unwrap_or_default(); panic!( "{}", serve_failure_evidence( world, + reported_engine(&stdout), &headline, &device_state, - world.cli_output.as_deref().unwrap_or_default(), - world.cli_stderr.as_deref().unwrap_or_default(), + &stdout, + &stderr, ) + .await ); } } @@ -2652,11 +2698,7 @@ async fn assert_next_automatic_port(world: &mut E2eWorld) { /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { - let Some(engine) = output - .lines() - .find_map(|l| l.trim().strip_prefix("engine:")) - .map(str::trim) - else { + let Some(engine) = reported_engine(output) else { panic!("no 'engine:' line in serve output:\n{output}"); }; engine