From 938d790c30837723c292a49c6b8bc2662aa4079e Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 26 Oct 2026 10:44:47 +0000 Subject: [PATCH 1/9] test(e2e): drive the Lemonade recovery scenario with a planted runtime `@id:serve-lemonade-preparation-recovery` (serve-18) needs Lemonade's llama.cpp backend install to fail twice, so it can pin the retry count and the recovery command. Two `#[cfg(feature = "e2e-test-hooks")]` branches provided that: one in the Lemonade engine's Install RPC that failed on `ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE`, and one in `rocm serve` that waived its no-GPU pre-flight for the same variable. Both are deleted. The `e2e-test-hooks` feature stays: it has other consumers (the simulated host root among them), and removing it is the last step of #349, not this one. The failure now happens where a broken install fails, at a non-zero child process: - `fake-lemonade`, a `[[bin]]` of the suite, is planted as the runtime's `lemonade` and `lemond`. `lemonade backends` reports `llamacpp:rocm` as installable and `lemonade backends install` exits non-zero; `lemond` stays alive until killed. - The install manifest is written beside it. Its `env_id` is derived from `rocm_deps::LEMONADE_VERSION` (a dev-dependency, so no crate-edge entry): `serve` skips its own install only when `Detect` reports exactly that id, and a placeholder silently runs the real multi-gigabyte install instead. - The pre-flight runs unchanged against one KFD GPU node planted under the simulated host root. The root holds nothing else, so the scenario needs no GPU and a GPU runner's devices do not take part. `XDG_RUNTIME_DIR` points inside the scenario too, so the fake server writes nothing outside it. The scenario is now `@requires-os:linux` instead of `@requires-no-gpu`. It still runs on the GPU-less every-PR lane, and now on the Linux GPU lanes and WSL2 as well. It stays off Windows: the host root re-roots Linux paths only, and there `rocm serve` does not notice a managed engine dying at startup (#467), so the failure would never reach the user. A test in `expectation.rs` pins that gate against the real tag line. The engine's unit test now reads the backends table out of the fixture's source with `include_str!`, instead of a copy kept in step by a comment. Workflow and contract-test comments that cited the deleted seam as their example now cite the host root. Checked by mutation: the scenario fails when the fake install succeeds, and fails with its named pre-flight message when the GPU is not planted. Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 6 +- Cargo.lock | 1 + apps/rocm/src/serve_cmd.rs | 42 ++++--- docs/architecture.md | 1 + engines/lemonade/src/lib.rs | 12 -- engines/lemonade/src/llamacpp_backend.rs | 34 ++++++ tests/e2e-cucumber/Cargo.toml | 12 ++ .../features/model_serving.feature | 17 ++- tests/e2e-cucumber/src/bin/fake-lemonade.rs | 105 ++++++++++++++++++ tests/e2e-cucumber/src/expectation.rs | 45 ++++++++ tests/e2e-cucumber/tests/e2e.rs | 100 +++++++++++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 23 ++-- xtask/src/workflow_contract.rs | 4 +- 13 files changed, 345 insertions(+), 57 deletions(-) create mode 100644 tests/e2e-cucumber/src/bin/fake-lemonade.rs diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 96b63f1b9c..e70daaf9e7 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -379,9 +379,9 @@ jobs: # superset of this one. `xtask/src/workflow_contract.rs` asserts the # floor for that reason. # - # The floor is what matters here. The suite's deterministic failure - # seams (e.g. the scripted Lemonade backend-install failure) are - # compiled out without it, so a pre-built binary that omits the feature + # The floor is what matters here. The suite's test seams (e.g. the + # simulated host root, ROCM_CLI_TEST_HOST_ROOT) are compiled out + # without it, so a pre-built binary that omits the feature # leaves those scenarios unable to reach their premise — they then fail # as regressions on whichever lane happens to select them. Every lane # that pre-builds and exports ROCM_CLI_BINARY must pass it. diff --git a/Cargo.lock b/Cargo.lock index ed1f160cf0..7dc244d234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,6 +1282,7 @@ dependencies = [ "rand 0.10.3", "reqwest 0.13.4", "rocm-core", + "rocm-deps", "serde", "serde_json", "sha2", diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2336899be7..49a1c924e4 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -420,13 +420,17 @@ fn aim_board_note( /// policy, on a host the probe *authoritatively* reports as having no usable /// AMD GPU. /// -/// Not "exactly when the bail fires", and the difference is only in test-hook -/// builds: the bail also requires `!scripted_backend_failure` (see the `if` -/// below it), which is `cfg!(feature = "e2e-test-hooks")`/env-gated. Under that -/// scenario the gate can open while the bail cannot fire. That is deliberate — -/// the hook stands in for the GPU the lane does not have, so the reuse path it -/// exercises must still behave as it would on a real GPU-less host — and it -/// costs nothing in a shipped binary, where the term can never be true. +/// Not "exactly when the bail fires". In every build the bail is also skipped +/// when this gate's own answer is a live service to reuse (`reuse_existing`). +/// E2E builds add two more skip terms (see the `if` below it), each env-gated +/// and compiled in only with its feature: `!e2e_simulate_oom_launch()` +/// (`e2e-oom-fault-injection`) stands in for the GPU the OOM scenario's lane +/// does not have, and `!scripted_free_space` (`e2e-test-hooks`) lets the +/// Lemonade free-space refusal, which `rocm serve` only reaches past the bail, +/// be exercised on a GPU-less lane. Under either hook the gate can open while +/// the bail cannot fire. That is deliberate — the pre-gate must still behave as +/// it would on a real GPU-less host — and it costs nothing in a shipped binary, +/// where neither hook term can be true. /// /// `host_has_no_usable_gpu` is therefore the bail's own predicate, computed once /// by the caller and handed to both, so the two cannot drift apart. They did: @@ -861,24 +865,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // // Skipped for cpu_only and when reusing an already-running service (nothing is // launched); permissive when availability cannot be probed on this platform - // (probe returns `None`). The E2E-only - // backend-failure scenario bypasses this host precondition so the black-box - // test reaches Lemonade's backend boundary without real GPU hardware, and the - // E2E-only OOM-launch fault injection likewise stands in for the GPU it does - // not have. + // (probe returns `None`). The E2E-only OOM-launch fault injection stands in + // for the GPU its lane does not have. // - // The scripted free-space override opens the same waiver: it exists to refuse a + // The scripted free-space override waives it too: it exists to refuse a // Lemonade install on a drive that cannot hold it, which `rocm serve` only // reaches past this bail. - let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") - && [ - "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", - "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", - ] - .iter() - .any(|variable| std::env::var_os(variable).is_some()); + let scripted_free_space = cfg!(feature = "e2e-test-hooks") + && std::env::var_os("ROCM_E2E_LEMONADE_FREE_SPACE_BYTES").is_some(); if !cpu_only - && !scripted_backend_failure + && !scripted_free_space && !e2e_simulate_oom_launch() && !reuse_existing && host_has_no_usable_gpu @@ -892,10 +888,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted - // backend failure is the one remaining term. Handed to the spawn, which + // free-space override is the one remaining term. Handed to the spawn, which // re-checks reuse authoritatively under the launch lock and raises the // refusal itself if the service is gone by then. - let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_backend_failure; + let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_free_space; // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via // `HIP_VISIBLE_DEVICES`; those orderings can diverge when // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. diff --git a/docs/architecture.md b/docs/architecture.md index b97561e7a3..18c9446d60 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -821,6 +821,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | File | Responsibility | Class | | --- | --- | --- | | [`src/bin/fake-docker.rs`](../tests/e2e-cucumber/src/bin/fake-docker.rs) | A `docker` stand-in that always fails, optionally logging its arguments, for the AIM Docker-unavailable scenarios. | flat | +| [`src/bin/fake-lemonade.rs`](../tests/e2e-cucumber/src/bin/fake-lemonade.rs) | The `lemonade` CLI and `lemond` server of a planted Lemonade runtime, chosen by `argv[0]`: `lemond` stays alive for a bounded time, and `lemonade backends install` fails, for `@id:serve-lemonade-preparation-recovery`. | flat | | [`src/bin/fake-python.rs`](../tests/e2e-cucumber/src/bin/fake-python.rs) | A stand-in for the runtime's Python that answers the ComfyUI install's torch-stack probe and post-install GPU check. | flat | | [`src/bin/fake-tailscale.rs`](../tests/e2e-cucumber/src/bin/fake-tailscale.rs) | A Tailscale CLI stand-in that serves the status document named by `FAKE_TAILSCALE_STATUS`, for the `rocm remote` scenarios. | flat | | [`src/bin/fake-uv.rs`](../tests/e2e-cucumber/src/bin/fake-uv.rs) | A `uv` stand-in whose output, exit code, argument log and stdin handling are set by `FAKE_UV_*` variables, for the ComfyUI scenarios. | flat | diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 78ea6c164c..f890fd9bba 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -57,8 +57,6 @@ pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; pub(crate) const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; pub(crate) const LLAMACPP_RECIPE: &str = "llamacpp"; pub(crate) const ROCM_BACKEND_NAME: &str = "rocm"; -#[cfg(feature = "e2e-test-hooks")] -const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE"; const DEFAULT_LOG_TAIL_LINES: usize = 200; /// How long a stop waits for the server to actually exit after each signal /// before reporting a timeout (or, under `force`, escalating to `SIGKILL`). @@ -398,16 +396,6 @@ fn install_and_record( fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; - // Debug builds expose a deterministic failure seam for the black-box CLI - // scenario that pins retry count and terminal recovery guidance. Keep it at - // the backend phase: the real defect happens after the embeddable is ready, - // and exercising it must not download or alter a runtime on the test host. - #[cfg(feature = "e2e-test-hooks")] - if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { - crate::llamacpp_backend::install_llamacpp_backend_with_retry(|| { - anyhow::bail!("scripted Lemonade backend install failure") - })?; - } let env_root = request .env_root .as_deref() diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 48a9caf91b..768958fbb6 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -394,6 +394,40 @@ mod tests { } } + /// The E2E fake runtime's `lemonade backends` table must drive this engine + /// to attempt a `llamacpp:rocm` install: that attempt is the whole premise + /// of `@id:serve-lemonade-preparation-recovery`, which then fails it. + /// + /// The table is read out of the fixture's own source rather than copied, so + /// editing it there is checked here, on every lane that runs unit tests. A + /// parser change that stopped accepting it would otherwise make the + /// scenario stop reaching the failure it asserts, with no unit test + /// noticing. + #[test] + fn the_e2e_fake_runtime_table_drives_a_rocm_backend_install() { + const FIXTURE: &str = include_str!("../../../tests/e2e-cucumber/src/bin/fake-lemonade.rs"); + let table = FIXTURE + .split_once("const BACKENDS_TABLE: &str = r\"") + .and_then(|(_, rest)| rest.split_once("\";")) + .map(|(table, _)| table) + .expect("fake-lemonade.rs must declare BACKENDS_TABLE as a raw string literal without hashes"); + let backends = parse_llamacpp_backend_statuses(table); + assert_eq!( + backends, + vec![ + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "unsupported".to_owned()), + ] + ); + // `false` = not already installed, which is what makes the engine run + // the install the fixture then refuses. `installed` would skip it, and + // the scenario would never see a failure. + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)) + ); + } + #[test] fn parses_llamacpp_backends_from_table() { let output = "\ diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 3efc130f5d..0b751f2b85 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -40,6 +40,13 @@ path = "src/bin/fake-python.rs" name = "fake-docker" path = "src/bin/fake-docker.rs" +# Stand-in for a Lemonade runtime's `lemonade` and `lemond` executables, planted +# by the serve-preparation-recovery scenario so Lemonade's backend install fails +# at a real child process rather than at a seam compiled into the engine. +[[bin]] +name = "fake-lemonade" +path = "src/bin/fake-lemonade.rs" + [dependencies] axum.workspace = true cucumber = { version = "0.23", features = ["output-json", "output-junit"] } @@ -100,6 +107,11 @@ rocm-core = { path = "../../crates/rocm-core" } # virtual clock that sleeps advance exactly, not the host timer, whose # granularity varies by platform (about 15.6 ms on Windows). tokio = { workspace = true, features = ["test-util"] } +# The pinned Lemonade version, so the planted Lemonade runtime's `env_id` is +# derived from the same constant `rocm` builds its expected runtime id from, +# rather than a copy a version bump would silently invalidate. Used only by the +# `e2e` test target, so a dev edge, which `xtask check-crate-edges` exempts. +rocm-deps = { path = "../../crates/rocm-deps" } # `fcntl(F_GETFL)` only, to read (never modify) the `O_NONBLOCK` state of the # suite's own stdio for the diagnostics log (see `src/harness_diagnostics.rs`). diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 3623c1e80a..671df0787d 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -197,12 +197,17 @@ Feature: Model serving Then serving is refused before any engine starts And the user is told the two selectors cannot be combined - # The failure is injected at Lemonade's backend-install boundary in debug/test - # builds, after the CLI has selected Lemonade but before any runtime download or - # machine mutation. That makes the user-visible retry and final recovery command - # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB - # transfer to fail at just the right moment. - @id:serve-lemonade-preparation-recovery @requires-no-gpu + # The harness plants a Lemonade runtime whose `lemonade backends install` + # always exits non-zero, so the failure happens at a real boundary -- a + # non-zero child process -- after the CLI has selected Lemonade but before any + # runtime download or machine mutation, and the retry and recovery command are + # deterministic without a real multi-gigabyte transfer failing at the right + # moment. `rocm serve` refuses before preparation on a host with no usable GPU, + # so the harness also plants one under a simulated host root, which re-roots + # Linux paths only; hence Linux. Windows is excluded for a second reason too: + # there `rocm serve` does not notice a managed engine dying at startup (#467), + # so the failure would not reach the user at all. + @id:serve-lemonade-preparation-recovery @requires-os:linux Scenario: serve-18 - Repeated Lemonade preparation failure gives the user a recovery path Given Lemonade preparation cannot complete When the user serves a model with Lemonade diff --git a/tests/e2e-cucumber/src/bin/fake-lemonade.rs b/tests/e2e-cucumber/src/bin/fake-lemonade.rs new file mode 100644 index 0000000000..51f8ff7c46 --- /dev/null +++ b/tests/e2e-cucumber/src/bin/fake-lemonade.rs @@ -0,0 +1,105 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in for the two executables a Lemonade runtime installs: the `lemonade` +//! CLI and the `lemond` server. +//! +//! `@id:serve-lemonade-preparation-recovery` needs Lemonade's llama.cpp backend +//! install to fail deterministically, twice, so the scenario can pin the retry +//! count and the terminal recovery guidance. That failure used to be a +//! `#[cfg(feature = "e2e-test-hooks")]` branch compiled into the Lemonade engine +//! itself, plus a second one in `rocm serve` that waived its no-GPU pre-flight. +//! Driving it from here keeps both out of the binary under test: the CLI spawns +//! these as real subprocesses over the interface it uses for a real runtime, and +//! only the runtime is fake. +//! +//! The harness copies this one binary into a planted runtime directory under +//! both names, so the role is taken from `argv[0]`: +//! +//! - `lemond` -- the server `rocm serve` spawns. It only has to stay alive; the +//! CLI's readiness check polls the `lemonade` CLI, not this process. +//! - `lemonade` -- the CLI the engine shells out to. Three calls matter, and the +//! real binary's contract for each is what this reproduces: +//! - `--host H --port P status` -> exit 0, which is how the engine decides the +//! server came up. +//! - `backends` -> [`BACKENDS_TABLE`], the recipe/backend/status table the +//! engine scrapes. Reporting `llamacpp rocm` as `installable` (not +//! `installed`) is what makes the engine attempt an install rather than +//! reuse one. +//! - `--host H --port P backends install llamacpp:rocm` -> non-zero, the +//! failure the scenario is about. + +use std::path::Path; +use std::process::ExitCode; +use std::time::Duration; + +/// What `lemonade backends` prints here. +/// +/// Read by the Lemonade engine's own unit tests (`include_str!` of this file), +/// which check that the engine parses it into a `llamacpp:rocm` install +/// attempt. Keep it a raw string literal without hashes: that test extracts it +/// textually. +const BACKENDS_TABLE: &str = r"Recipe Backend Status +-------- -------- ----------- +llamacpp rocm installable +llamacpp vulkan unsupported +"; + +fn main() -> ExitCode { + // Copied rather than symlinked (self-hosted Windows runners commonly lack + // SeCreateSymbolicLinkPrivilege), so the file name is the only role signal. + let argv0 = std::env::args_os().next().unwrap_or_default(); + let role = Path::new(&argv0) + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) + .unwrap_or_default(); + let args: Vec = std::env::args().skip(1).collect(); + + match role.as_str() { + "lemond" => run_lemond(), + "lemonade" => run_lemonade_cli(&args), + other => { + eprintln!( + "fake-lemonade: copied to an unexpected name {other:?}; expected `lemond` or \ + `lemonade`" + ); + ExitCode::FAILURE + } + } +} + +/// Stay alive until the parent kills us. `rocm serve` spawns `lemond`, records +/// its pid, and terminates it when the serve fails -- so exiting on our own +/// would race that teardown and turn the scenario's expected backend-install +/// failure into a spurious "server exited" diagnosis. +fn run_lemond() -> ExitCode { + loop { + std::thread::sleep(Duration::from_mins(1)); + } +} + +fn run_lemonade_cli(args: &[String]) -> ExitCode { + if let Some(index) = args.iter().position(|arg| arg == "backends") { + if args.get(index + 1).map(String::as_str) == Some("install") { + // Fail loudly on stderr: the engine surfaces the child's output, and + // a scenario that somehow ran against a real runtime should be + // obvious in the log rather than look like a genuine install bug. + eprintln!( + "fake-lemonade: refusing to install a backend; this runtime was planted by the \ + E2E harness for @id:serve-lemonade-preparation-recovery" + ); + return ExitCode::FAILURE; + } + print!("{BACKENDS_TABLE}"); + return ExitCode::SUCCESS; + } + + if args.iter().any(|arg| arg == "status") { + return ExitCode::SUCCESS; + } + + // Any other subcommand: succeed quietly. The scenario never reaches model + // load, and failing here would mask the failure it is actually asserting. + ExitCode::SUCCESS +} diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 9ebdca6c6b..1fdcb3f44e 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1556,6 +1556,51 @@ serve_timeout_secs = 90 } } + /// `serve-18`'s gate decides which lanes exercise Lemonade's preparation + /// retry, so it is pinned against the REAL tag line: a tag edit must fail a + /// test rather than quietly move the scenario. + /// + /// It runs on every Linux lane, GPU or not -- the harness plants the GPU the + /// pre-flight needs under a simulated host root -- which keeps the cadence + /// the old compiled-in seam gave it on the GPU-less every-PR lane. It must + /// not run on Windows, where the host root re-roots nothing and `rocm serve` + /// does not notice a managed engine dying at startup (#467). + #[test] + fn the_lemonade_recovery_scenario_runs_on_every_linux_lane_and_not_on_windows() { + let feature = include_str!("../features/model_serving.feature"); + let m = Expectations::default(); + let decl = ScenarioDecl::from_tags(&feature_tags( + feature, + "serve-lemonade-preparation-recovery", + )); + let gpu_less_linux = HostCapability { + gfx_target: None, + has_amd_gpu: false, + amd_gpu_count: Some(0), + platform_slug: "mock".into(), + ..cap("strix-ubuntu") + }; + for (host, capability) in [ + ("GPU-less Linux", gpu_less_linux), + ("strix-ubuntu", cap("strix-ubuntu")), + ("mi300x", cap("mi300x")), + ("wsl2", cap("wsl2")), + ] { + assert_eq!( + resolve(&decl, &capability, &m, Included::default()), + Expectation::ExpectPass, + "{host} must run serve-18" + ); + } + assert!( + matches!( + resolve(&decl, &cap("strix-windows"), &m, Included::default()), + Expectation::Skip { .. } + ), + "Windows must skip serve-18" + ); + } + /// The single-GPU skip must name the count, not just repeat the `@requires-gpu` /// reason: on Strix Halo a GPU *is* present, so "none detected" would send a /// reader of the report looking for a missing device that is right there. diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6e8db6c579..8fb0e5404c 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -686,6 +686,106 @@ impl E2eWorld { self.legacy_rocm_path = Some(rocm); } + /// Plant a Lemonade runtime whose llama.cpp backend install always fails, on + /// a simulated host with one AMD GPU, so + /// `@id:serve-lemonade-preparation-recovery` can pin the retry count and the + /// terminal recovery guidance without a real GPU or a multi-gigabyte + /// runtime download. + /// + /// This replaces two `#[cfg(feature = "e2e-test-hooks")]` branches: one in + /// the Lemonade engine that failed the install on an environment variable, + /// and one in `rocm serve` that waived its no-GPU pre-flight for the same + /// variable. The failure now happens at the boundary a broken install fails + /// at -- a non-zero `lemonade backends install` -- and the pre-flight runs + /// unchanged against a host that has a GPU. + /// + /// **The runtime.** Black-box, like [`Self::register_mock_service`]: the + /// manifest is plain JSON matching the engine's on-disk schema, not a typed + /// import. `resolve_runtime` accepts it because it checks only that the + /// manifest parses and that `lemond` is a file; no checksum, version or + /// signature is on this path, so nothing here weakens a check the product + /// performs. `env_id` is the one field that cannot be a sentinel: `serve` + /// skips its own install only when `Detect` reports the id + /// `managed_engine_runtime_id` builds for the pinned version, and a + /// placeholder makes it quietly run the real install instead. So it is + /// derived from `rocm_deps` rather than copied. + /// + /// It goes under the scenario's own `data/engines/lemonade`, because this + /// scenario does not call [`Self::use_shared_runtimes`]: with an empty + /// runtimes registry the CLI resolves no env root, so `lemonade_root` is the + /// isolated engine dir. + /// + /// **The GPU.** One KFD GPU node under a simulated host root + /// (`ROCM_CLI_TEST_HOST_ROOT`), which is what `rocm serve`'s pre-flight + /// counts. The root holds nothing else, so a GPU runner's real devices do + /// not take part either. The host root is honoured only by the + /// `e2e-test-hooks` build and re-roots Linux paths only, which is why the + /// scenario is `@requires-os:linux`. The serve must therefore run through + /// `run_rocm_with_scenario_env`. + pub fn plant_failing_lemonade_runtime(&mut self) { + let root = self + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let engine = root.join("data").join("engines").join("lemonade"); + let runtime_dir = engine.join("runtime"); + // The fixture binaries go under `runtime/` and the manifest under + // `manifests/`; `resolve_runtime` reads that manifest and requires its + // `lemond` to be a file. The engine's launch creates its own state and + // log directories, so they are not planted here. + for dir in [&runtime_dir, &engine.join("manifests")] { + std::fs::create_dir_all(dir) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", dir.display())); + } + + // One fixture under both names; it takes its role from argv[0]. + let lemond = runtime_dir.join(e2e_cucumber::fixture_binary::fixture_target_name("lemond")); + let lemonade = runtime_dir.join(e2e_cucumber::fixture_binary::fixture_target_name( + "lemonade", + )); + install_fixture_binary("fake-lemonade", &lemond); + install_fixture_binary("fake-lemonade", &lemonade); + + let manifest = serde_json::json!({ + "env_id": format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + "version": rocm_deps::LEMONADE_VERSION, + "runtime_dir": runtime_dir, + "lemond": lemond, + "lemonade": lemonade, + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + }); + let path = engine.join("manifests").join("runtime.json"); + std::fs::write( + &path, + serde_json::to_vec_pretty(&manifest).expect("manifest json"), + ) + .unwrap_or_else(|e| panic!("failed to write {}: {e}", path.display())); + + let host_root = root.join("host-root"); + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + "cpu_cores_count 0\nsimd_count 120\ngfx_target_version 110001\n", + ) + .expect("failed to write planted KFD node properties"); + self.command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); + + // The engine prepares a runtime directory for `lemond` under + // `XDG_RUNTIME_DIR`, falling back to the real `$HOME`. Give it one inside + // the scenario, so the fake server writes nothing outside the isolated + // tree and a runner whose `$HOME` is not writable still gets this far. + let runtime = root.join("runtime"); + std::fs::create_dir_all(&runtime).expect("failed to create isolated runtime dir"); + self.command_env + .push(("XDG_RUNTIME_DIR", runtime.into_os_string())); + } + /// Register the running mock server with the CLI by writing a managed-service /// record into the isolated services directory (`/services/`), exactly /// as `rocm serve --managed` would. This lets `rocm services list` and the diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9f5f4a6820..dc44c218c5 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1401,10 +1401,10 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { #[given("Lemonade preparation cannot complete")] async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) { - world.command_env.push(( - "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", - "repeated".into(), - )); + // A planted runtime whose `lemonade backends install` always exits + // non-zero, on a simulated host with one GPU: the CLI fails at the boundary + // a broken install fails at, rather than at a branch compiled into it. + world.plant_failing_lemonade_runtime(); } #[when("the user serves a model with Lemonade")] @@ -2485,15 +2485,16 @@ async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { Some(0), "serve unexpectedly succeeded:\n{output}" ); - // The scripted failure seam also waives serve's no-GPU pre-flight, so this - // refusal can only appear when the seam is compiled out. Name that cause: - // otherwise a lane that pre-builds `rocm` without the feature reports a - // baffling "no retry announcement" instead of its real misconfiguration. + // The pre-flight counts the GPU planted under the simulated host root, which + // only an `e2e-test-hooks` build honours. Name that cause: otherwise a lane + // that pre-builds `rocm` without the feature reports a baffling "no retry + // announcement" instead of its real misconfiguration. assert!( !output.contains("no usable AMD GPU detected"), - "serve stopped at the no-GPU pre-flight, so the binary under test was \ - built without the `rocm/e2e-test-hooks` feature and never reached \ - Lemonade preparation. E2E lanes that pre-build `rocm` and export \ + "serve stopped at the no-GPU pre-flight, so the binary under test did not \ + see the GPU planted under ROCM_CLI_TEST_HOST_ROOT and never reached \ + Lemonade preparation. It was probably built without the \ + `rocm/e2e-test-hooks` feature: E2E lanes that pre-build `rocm` and export \ ROCM_CLI_BINARY must pass `--features rocm/e2e-test-hooks`, matching \ what `cargo xtask e2e` builds for itself:\n{output}" ); diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b0..7ca357400c 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -269,8 +269,8 @@ mod tests { /// Every lane that pre-builds `rocm` and hands it to the suite via /// `ROCM_CLI_BINARY` must enable `rocm/e2e-test-hooks`. /// - /// The suite's deterministic failure seams (e.g. the scripted Lemonade - /// backend-install failure) are `#[cfg(feature = "e2e-test-hooks")]`. A lane + /// The suite's test seams (e.g. the simulated host root, + /// `ROCM_CLI_TEST_HOST_ROOT`) are `#[cfg(feature = "e2e-test-hooks")]`. A lane /// that omits the feature ships a binary in which those seams do not exist, /// so the scenarios relying on them cannot reach their premise and fail as /// regressions — but only on whichever lane happens to select them, which is From b179633a71adc15d454edba596e4208a43bdf801 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 13:43:59 +0200 Subject: [PATCH 2/9] test(e2e): bound the fake lemond and prove serve-18 used the planted runtime The fake `lemond` now exits once its parent is gone (Unix re-parenting) and after a 10-minute backstop, so a run whose best-effort teardown fails leaves no process behind while #729 is open. serve-18 now asserts that the planted runtime refused both backend installs, through a constant shared between the fixture and the step. If the planted env_id ever stopped matching Detect, the scenario now fails fast instead of starting the real multi-gigabyte install. Also fixes a stale "scripted failure seams" assert message and lists fake-lemonade among the README's fixture binaries. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/README.md | 4 +- tests/e2e-cucumber/src/bin/fake-lemonade.rs | 42 ++++++++++++++----- tests/e2e-cucumber/src/fixture_binary.rs | 10 +++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 10 +++++ xtask/src/workflow_contract.rs | 4 +- 5 files changed, 56 insertions(+), 14 deletions(-) diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d005021bc3..0475468e83 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -255,7 +255,9 @@ The `.feature` file is both the spec and the test input — cucumber reads it at [`tests/e2e.rs`](tests/e2e.rs) plants it (it needs the `rocm-core` dev-dependency the library target can't reach). See [`fake-uv`](src/bin/fake-uv.rs), [`fake-python`](src/bin/fake-python.rs), - and [`fake-tailscale`](src/bin/fake-tailscale.rs) for the pattern. + [`fake-tailscale`](src/bin/fake-tailscale.rs), and + [`fake-lemonade`](src/bin/fake-lemonade.rs) (one binary planted under two + names, taking its role from `argv[0]`) for the pattern. ## Design principles diff --git a/tests/e2e-cucumber/src/bin/fake-lemonade.rs b/tests/e2e-cucumber/src/bin/fake-lemonade.rs index 51f8ff7c46..6756a5b466 100644 --- a/tests/e2e-cucumber/src/bin/fake-lemonade.rs +++ b/tests/e2e-cucumber/src/bin/fake-lemonade.rs @@ -34,6 +34,8 @@ use std::path::Path; use std::process::ExitCode; use std::time::Duration; +use e2e_cucumber::fixture_binary::FAKE_LEMONADE_INSTALL_REFUSAL; + /// What `lemonade backends` prints here. /// /// Read by the Lemonade engine's own unit tests (`include_str!` of this file), @@ -69,25 +71,43 @@ fn main() -> ExitCode { } } -/// Stay alive until the parent kills us. `rocm serve` spawns `lemond`, records -/// its pid, and terminates it when the serve fails -- so exiting on our own -/// would race that teardown and turn the scenario's expected backend-install -/// failure into a spurious "server exited" diagnosis. +/// Stay alive for as long as the process that spawned us does. +/// +/// Exiting while the engine still runs would turn the scenario's expected +/// backend-install failure into a spurious "server exited" diagnosis. But the +/// engine does not stop `lemond` when the backend install fails (#729): it +/// exits and leaves the server running, and only the scenario's teardown +/// (`stop_managed_services`, best-effort) stops it. So this fake also exits on +/// its own once its parent is gone -- on Unix, when it has been re-parented -- +/// and in any case after [`LEMOND_MAX_LIFETIME`], so a run whose teardown did +/// not get that far still leaves no process behind. fn run_lemond() -> ExitCode { - loop { - std::thread::sleep(Duration::from_mins(1)); + #[cfg(unix)] + let spawned_by = std::os::unix::process::parent_id(); + let started = std::time::Instant::now(); + while started.elapsed() < LEMOND_MAX_LIFETIME { + std::thread::sleep(Duration::from_millis(500)); + #[cfg(unix)] + if std::os::unix::process::parent_id() != spawned_by { + break; + } } + ExitCode::SUCCESS } +/// Backstop on the fake server's lifetime, for a parent that never exits or a +/// platform where re-parenting is not observable. The scenario finishes in +/// seconds, so this is far longer than any run needs. +const LEMOND_MAX_LIFETIME: Duration = Duration::from_mins(10); + fn run_lemonade_cli(args: &[String]) -> ExitCode { if let Some(index) = args.iter().position(|arg| arg == "backends") { if args.get(index + 1).map(String::as_str) == Some("install") { - // Fail loudly on stderr: the engine surfaces the child's output, and - // a scenario that somehow ran against a real runtime should be - // obvious in the log rather than look like a genuine install bug. + // Fail loudly on stderr: the engine passes the child's stderr + // through, and the scenario asserts this line to prove the install + // it saw fail was this fake's rather than a real runtime's. eprintln!( - "fake-lemonade: refusing to install a backend; this runtime was planted by the \ - E2E harness for @id:serve-lemonade-preparation-recovery" + "{FAKE_LEMONADE_INSTALL_REFUSAL} for @id:serve-lemonade-preparation-recovery" ); return ExitCode::FAILURE; } diff --git a/tests/e2e-cucumber/src/fixture_binary.rs b/tests/e2e-cucumber/src/fixture_binary.rs index fa055fa28c..cb90028201 100644 --- a/tests/e2e-cucumber/src/fixture_binary.rs +++ b/tests/e2e-cucumber/src/fixture_binary.rs @@ -19,6 +19,16 @@ use std::path::PathBuf; +/// What the `fake-lemonade` stand-in prints on stderr each time it refuses a +/// `lemonade backends install`. +/// +/// The Lemonade recovery scenario asserts it in `rocm serve`'s output, which +/// proves the install the CLI attempted was the planted runtime's. If the +/// planted manifest stopped matching what `Detect` expects, `serve` would run +/// the real runtime install instead, and this line would be missing. +pub const FAKE_LEMONADE_INSTALL_REFUSAL: &str = + "fake-lemonade: refusing to install a backend; this runtime was planted by the E2E harness"; + /// The path cargo built the `[[bin]]` named `name` at, alongside this test /// binary. pub fn fixture_binary(name: &str) -> PathBuf { diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index dc44c218c5..aee42727be 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2503,6 +2503,16 @@ async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { 1, "expected exactly one retry announcement:\n{output}" ); + // Both attempts must have been refused by the planted runtime. Without its + // line, `serve` did not accept the planted runtime and attempted some other + // install -- for a mismatched `env_id`, the real multi-gigabyte one. + assert_eq!( + output + .matches(e2e_cucumber::fixture_binary::FAKE_LEMONADE_INSTALL_REFUSAL) + .count(), + 2, + "expected the planted runtime to refuse both backend installs:\n{output}" + ); } #[then("the user is told how to reinstall Lemonade and retry serving")] diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 7ca357400c..3db4e4e700 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -300,8 +300,8 @@ mod tests { ), "{workflow} prebuilt E2E lane must build with \ `--features rocm/e2e-test-hooks`, the hook floor `cargo xtask e2e` \ - also builds for itself; without it the suite's scripted failure \ - seams are compiled out:\n{block}" + also builds for itself; without it the suite's test seams, such as \ + the simulated host root, are compiled out:\n{block}" ); } } From b7ea0c315aba980dbe90f45aa302a0b78af3670e Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 10 Nov 2026 12:35:58 +0200 Subject: [PATCH 3/9] test(e2e): read the Lemonade pin as data instead of importing rocm-deps The e2e suite is black-box and imports nothing from the rocm-cli codebase, per its README. The planted Lemonade runtime's env_id was the exception: it took the version from a rocm-deps dev-dependency. It now reads the version from runtime-deps.toml, the workspace pin file that rocm-deps' build script generates LEMONADE_VERSION from, so the planted id still follows a version bump with no copy to keep in step. A missing or misspelled pin panics with the file's path. The README's black-box principle names this as the way a scenario gets a pinned version. Signed-off-by: Roman Inflianskas --- Cargo.lock | 1 - docs/architecture.md | 1 + tests/e2e-cucumber/Cargo.toml | 5 -- tests/e2e-cucumber/README.md | 2 +- tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/runtime_pins.rs | 88 ++++++++++++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 10 +-- 7 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 tests/e2e-cucumber/src/runtime_pins.rs diff --git a/Cargo.lock b/Cargo.lock index 7dc244d234..ed1f160cf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1282,7 +1282,6 @@ dependencies = [ "rand 0.10.3", "reqwest 0.13.4", "rocm-core", - "rocm-deps", "serde", "serde_json", "sha2", diff --git a/docs/architecture.md b/docs/architecture.md index 18c9446d60..e8022f0553 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -807,6 +807,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/quit_and_wait.rs`](../tests/e2e-cucumber/src/quit_and_wait.rs) | Retry loop for quitting a TUI that may open a quit-confirm prompt first. | flat | | [`src/reader_failure.rs`](../tests/e2e-cucumber/src/reader_failure.rs) | Failure state shared between a background terminal reader and its waiters. | flat | | [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | +| [`src/runtime_pins.rs`](../tests/e2e-cucumber/src/runtime_pins.rs) | Reads a pinned runtime version out of the workspace-root [`runtime-deps.toml`](../runtime-deps.toml) as plain data, so a planted runtime keyed by that pin (the Lemonade `env_id`) follows a version bump without importing `rocm-deps`. | flat | | [`src/send_until.rs`](../tests/e2e-cucumber/src/send_until.rs) | Retry loop for sending an idempotent terminal input until its effect appears. | flat | | [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail. | flat | | [`src/serve_plan.rs`](../tests/e2e-cucumber/src/serve_plan.rs) | Explains why a `rocm serve` run printed no `resolved model:` line by keeping stderr's refusal alongside stdout. | flat | diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 0b751f2b85..ce25d46c18 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -107,11 +107,6 @@ rocm-core = { path = "../../crates/rocm-core" } # virtual clock that sleeps advance exactly, not the host timer, whose # granularity varies by platform (about 15.6 ms on Windows). tokio = { workspace = true, features = ["test-util"] } -# The pinned Lemonade version, so the planted Lemonade runtime's `env_id` is -# derived from the same constant `rocm` builds its expected runtime id from, -# rather than a copy a version bump would silently invalidate. Used only by the -# `e2e` test target, so a dev edge, which `xtask check-crate-edges` exempts. -rocm-deps = { path = "../../crates/rocm-deps" } # `fcntl(F_GETFL)` only, to read (never modify) the `O_NONBLOCK` state of the # suite's own stdio for the diagnostics log (see `src/harness_diagnostics.rs`). diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 0475468e83..4e4c30bc7e 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -261,7 +261,7 @@ The `.feature` file is both the spec and the test input — cucumber reads it at ## Design principles -- **Black-box only.** Step functions interact with `rocm` and explicitly configured `rocmd` through their CLIs and HTTP endpoints. No imports from the rocm-cli codebase. Where a scenario needs the CLI to know about a running server, it plants a managed-service record as plain JSON matching the on-disk schema (see `register_mock_service`) — the same file `rocm serve --managed` would write — rather than importing the record type. (A harness *unit* test may read a planted record back through `rocm_core::ManagedServiceRecord`, to tie the writer's field names to the ones the CLI reads; steps do not.) +- **Black-box only.** Step functions interact with `rocm` and explicitly configured `rocmd` through their CLIs and HTTP endpoints. No imports from the rocm-cli codebase. Where a scenario needs the CLI to know about a running server, it plants a managed-service record as plain JSON matching the on-disk schema (see `register_mock_service`) — the same file `rocm serve --managed` would write — rather than importing the record type. (A harness *unit* test may read a planted record back through `rocm_core::ManagedServiceRecord`, to tie the writer's field names to the ones the CLI reads; steps do not.) Where a scenario plants something keyed by a pinned runtime version, such as the Lemonade runtime's `env_id`, it reads the version from the workspace pin file [`runtime-deps.toml`](../../runtime-deps.toml) as data (see [`runtime_pins`](src/runtime_pins.rs)) rather than importing `rocm-deps`. - **Isolated state.** Each scenario uses isolated config, data, and cache directories. Tests never touch `~/.rocm`. - **Behavioral language.** Feature files describe what users care about, not implementation details. How steps are implemented (mock vs real, which port, which API) stays in the step functions. - **A scenario about a sequence of chat completions scripts them.** `MockServer::chat_requests` records every chat completion request body even with the fixed stub reply, so a scenario can count completions and read each prompt. A script makes a double ask visible in what the user sees as well: `MockServer::start_with_chat_script` hands out one distinct reply per chat completion request (on either the `/v1/chat/completions` or the unversioned `/chat/completions` route), so the plan that gets printed or returned shows which call produced it, and a request past the end of the script gets HTTP 500 (and is still recorded), so asking more often than scripted fails loudly instead of recycling a reply. Plant the service with `ServiceRecordOptions { inference_verified: true, .. }` so the CLI's own readiness probe does not consume the first scripted reply. `planner_steps.rs` is the worked example. diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index b1508b5c64..5c2f852476 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -27,6 +27,7 @@ pub mod polled_deadline; pub mod powershell_text_rule; pub mod quit_and_wait; pub mod reader_failure; +pub mod runtime_pins; pub mod runtime_root; pub mod send_until; pub mod serve_log; diff --git a/tests/e2e-cucumber/src/runtime_pins.rs b/tests/e2e-cucumber/src/runtime_pins.rs new file mode 100644 index 0000000000..da0ab5d679 --- /dev/null +++ b/tests/e2e-cucumber/src/runtime_pins.rs @@ -0,0 +1,88 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The pinned third-party runtime versions, read from the workspace-root +//! `runtime-deps.toml` as plain data. +//! +//! That file is the single source of truth for the pins: the product turns it +//! into constants at build time, in the `rocm-deps` crate. The suite is +//! black-box and imports nothing from the rocm-cli codebase (see the README's +//! design principles), so it reads the same file instead. A scenario that must +//! plant something keyed by a pinned version, such as the Lemonade runtime's +//! `env_id`, then follows a version bump with no copy to keep in step. + +use std::path::{Path, PathBuf}; + +/// The pin file, relative to this crate's manifest directory. +const PINS_FILE: &str = "../../runtime-deps.toml"; + +/// Path of the workspace-root pin file. +pub fn pins_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(PINS_FILE) +} + +/// The `version` pinned for `[runtime.]` in the workspace pin file. +/// +/// Panics if the file cannot be read or has no such pin: a scenario that plants +/// a runtime under a missing or misspelled pin must fail loudly, not plant one +/// the CLI does not recognise. +pub fn pinned_version(runtime: &str) -> String { + let path = pins_path(); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + version_in(&text, runtime).unwrap_or_else(|e| panic!("{}: {e}", path.display())) +} + +/// The `version` field of `[runtime.]` in a pin file's text. +pub fn version_in(text: &str, runtime: &str) -> Result { + let doc: toml::Table = toml::from_str(text).map_err(|e| format!("not valid TOML: {e}"))?; + doc.get("runtime") + .and_then(toml::Value::as_table) + .and_then(|runtimes| runtimes.get(runtime)) + .and_then(|entry| entry.get("version")) + .and_then(toml::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| format!("no string `version` under [runtime.{runtime}]")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_lemonade_pin_from_the_workspace_file() { + let version = pinned_version("lemonade"); + // The pin is the value the file holds under `[runtime.lemonade]`, read + // independently of `version_in`, so a parser that ignored the runtime + // name or returned a constant would fail here. + let text = std::fs::read_to_string(pins_path()).expect("pin file is readable"); + let in_section = text + .split("[runtime.lemonade]") + .nth(1) + .expect("pin file has a [runtime.lemonade] table"); + let line = in_section + .lines() + .find(|line| line.trim_start().starts_with("version")) + .expect("[runtime.lemonade] has a version line"); + assert!( + line.contains(&format!("\"{version}\"")), + "pinned_version returned {version:?}, the file says {line:?}" + ); + // Same shape the product's own pin check requires: a bare version. + assert!( + !version.is_empty() && !version.starts_with('v'), + "pin must be a bare version, got {version:?}" + ); + } + + #[test] + fn picks_the_named_runtime_and_rejects_a_missing_one() { + let text = + "[runtime.lemonade]\nversion = \"1.2.3\"\n\n[runtime.other]\nversion = \"9.9.9\"\n"; + assert_eq!(version_in(text, "lemonade").as_deref(), Ok("1.2.3")); + assert_eq!(version_in(text, "other").as_deref(), Ok("9.9.9")); + assert!(version_in(text, "absent").is_err()); + assert!(version_in("[runtime.lemonade]\n", "lemonade").is_err()); + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 8fb0e5404c..1562198503 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -707,8 +707,9 @@ impl E2eWorld { /// performs. `env_id` is the one field that cannot be a sentinel: `serve` /// skips its own install only when `Detect` reports the id /// `managed_engine_runtime_id` builds for the pinned version, and a - /// placeholder makes it quietly run the real install instead. So it is - /// derived from `rocm_deps` rather than copied. + /// placeholder makes it quietly run the real install instead. So its + /// version is read from the workspace pin file, `runtime-deps.toml`, the + /// same file the product builds that id from, rather than copied. /// /// It goes under the scenario's own `data/engines/lemonade`, because this /// scenario does not call [`Self::use_shared_runtimes`]: with an empty @@ -748,9 +749,10 @@ impl E2eWorld { install_fixture_binary("fake-lemonade", &lemond); install_fixture_binary("fake-lemonade", &lemonade); + let version = e2e_cucumber::runtime_pins::pinned_version("lemonade"); let manifest = serde_json::json!({ - "env_id": format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), - "version": rocm_deps::LEMONADE_VERSION, + "env_id": format!("lemonade-embeddable-{version}"), + "version": version, "runtime_dir": runtime_dir, "lemond": lemond, "lemonade": lemonade, From 895c4fb096c9a8fd74cc0893c23e8fa2ca1ec7c1 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 10 Nov 2026 12:36:42 +0200 Subject: [PATCH 4/9] test(e2e): share the fake Lemonade backends table as a fixture file The engine's unit test pulled the fake runtime's `lemonade backends` table out of fake-lemonade.rs by matching the text of a raw-string literal, so it depended on how that literal was spelled. The table is now a data file under the e2e fixtures. The fake binary prints it with include_str!, and the engine test reads the same file, as rocm-core's fix tests already read the PowerShell cases there. The pre-push hook lists the table as read by rocm-engine-lemonade, so editing it runs that crate's tests. Also drops three comments that still described serve-18's removed failure hook as waiving the no-GPU pre-flight. Since the rebase, the free-space override is the only scripted waiver left. Signed-off-by: Roman Inflianskas --- docs/testing.md | 7 ++++--- engines/lemonade/src/llamacpp_backend.rs | 20 ++++++++----------- .../features/lemonade_installs.feature | 2 +- .../fixtures/lemonade_backends_table.txt | 4 ++++ tests/e2e-cucumber/src/bin/fake-lemonade.rs | 13 ++++-------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 4 ++-- xtask/src/prepush.rs | 11 ++++++++-- 7 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 tests/e2e-cucumber/fixtures/lemonade_backends_table.txt diff --git a/docs/testing.md b/docs/testing.md index d5b956cea8..6f2ff44cbb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -992,8 +992,9 @@ pushed (`PRE_COMMIT_FROM_REF..PRE_COMMIT_TO_REF`, the commits the remote does no The directory-based selection cannot see a crate's tests reading a file from outside the crate, so the hook widens it: any Rust file also selects `xtask` (its contract tests scan every source file), `README.md`, `docs/` and `skills/` also select `rocm` and `xtask` (the advised-commands -test reads them and `rocm` compiles in the assistant skill), and the e2e PowerShell fixture also -selects `rocm-core` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in +test reads them and `rocm` compiles in the assistant skill), and the e2e `fixtures/` directory also +selects `rocm-core` (its tests `include_str!` the PowerShell cases), and the fake Lemonade runtime's +backends table there also selects `rocm-engine-lemonade` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in [prepush.rs](../xtask/src/prepush.rs), and a unit test scans the workspace for `include_str!`, `include_bytes!` and manifest-relative reads that leave their crate and fails when one is not covered, so a new cross-crate read cannot be added without a decision. The scan has a limit: a read @@ -1943,7 +1944,7 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES`, compiled only into the `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the disk that nothing was downloaded. That variable also waives serve's no-GPU - refusal, as serve-18's failure hook does. + refusal, so the scenario reaches the install on a host with no GPU. - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 768958fbb6..20710c8b44 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -398,20 +398,16 @@ mod tests { /// to attempt a `llamacpp:rocm` install: that attempt is the whole premise /// of `@id:serve-lemonade-preparation-recovery`, which then fails it. /// - /// The table is read out of the fixture's own source rather than copied, so - /// editing it there is checked here, on every lane that runs unit tests. A - /// parser change that stopped accepting it would otherwise make the - /// scenario stop reaching the failure it asserts, with no unit test - /// noticing. + /// The fixture prints the table from a data file, and this test reads the + /// same file, so editing it there is checked here, on every lane that runs + /// unit tests. A parser change that stopped accepting it would otherwise + /// make the scenario stop reaching the failure it asserts, with no unit + /// test noticing. #[test] fn the_e2e_fake_runtime_table_drives_a_rocm_backend_install() { - const FIXTURE: &str = include_str!("../../../tests/e2e-cucumber/src/bin/fake-lemonade.rs"); - let table = FIXTURE - .split_once("const BACKENDS_TABLE: &str = r\"") - .and_then(|(_, rest)| rest.split_once("\";")) - .map(|(table, _)| table) - .expect("fake-lemonade.rs must declare BACKENDS_TABLE as a raw string literal without hashes"); - let backends = parse_llamacpp_backend_statuses(table); + const TABLE: &str = + include_str!("../../../tests/e2e-cucumber/fixtures/lemonade_backends_table.txt"); + let backends = parse_llamacpp_backend_statuses(TABLE); assert_eq!( backends, vec![ diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index e55ad7fe16..24d0badd39 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -27,7 +27,7 @@ Feature: Lemonade installs on disk # `rocm serve` reaches the same install on its own, which is how the second # copy used to appear without a word. The free-space override also stands in - # for the GPU the no-GPU lane does not have, as serve-18's failure hook does. + # for the GPU the no-GPU lane does not have. @id:lemonade-installs-serve-refused-without-space Scenario: lemonade-installs-02 - Serving with Lemonade on a drive that cannot hold it is refused before downloading Given the drive for a new Lemonade install cannot hold even Lemonade itself diff --git a/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt b/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt new file mode 100644 index 0000000000..44410af293 --- /dev/null +++ b/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt @@ -0,0 +1,4 @@ +Recipe Backend Status +-------- -------- ----------- +llamacpp rocm installable +llamacpp vulkan unsupported diff --git a/tests/e2e-cucumber/src/bin/fake-lemonade.rs b/tests/e2e-cucumber/src/bin/fake-lemonade.rs index 6756a5b466..40c4382fd3 100644 --- a/tests/e2e-cucumber/src/bin/fake-lemonade.rs +++ b/tests/e2e-cucumber/src/bin/fake-lemonade.rs @@ -38,15 +38,10 @@ use e2e_cucumber::fixture_binary::FAKE_LEMONADE_INSTALL_REFUSAL; /// What `lemonade backends` prints here. /// -/// Read by the Lemonade engine's own unit tests (`include_str!` of this file), -/// which check that the engine parses it into a `llamacpp:rocm` install -/// attempt. Keep it a raw string literal without hashes: that test extracts it -/// textually. -const BACKENDS_TABLE: &str = r"Recipe Backend Status --------- -------- ----------- -llamacpp rocm installable -llamacpp vulkan unsupported -"; +/// A data file rather than a literal so the Lemonade engine's own unit tests +/// can `include_str!` the same file and check that the engine parses it into a +/// `llamacpp:rocm` install attempt. +const BACKENDS_TABLE: &str = include_str!("../../fixtures/lemonade_backends_table.txt"); fn main() -> ExitCode { // Copied rather than symlinked (self-hosted Windows runners commonly lack diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index aee42727be..a4da6e8a6c 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2160,8 +2160,8 @@ async fn assert_no_runtime_manifest_written(world: &mut E2eWorld) { /// Serve a Lemonade model under the GPU-required default while the unrelated /// live service planted above exists. No fault-injection env var: the scripted -/// Lemonade backend failure would waive the very GPU pre-flight this scenario -/// asserts. +/// Lemonade free-space override would waive the very GPU pre-flight this +/// scenario asserts. #[when("the user serves a different model with Lemonade under the GPU-required default")] async fn user_serves_other_model_with_unrelated_service_live(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm( diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 9543c92115..f93e8281fa 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -211,14 +211,21 @@ fn package_args<'a>(packages: Option>) -> Vec bool { From 90dee28c7c9f5ecdef20820b9ac368d49cd14051 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 18:51:54 +0200 Subject: [PATCH 5/9] test(e2e): answer the fake lemonade call by its subcommand position The fake matched `status` and `backends` anywhere in its arguments, so an argument that only equalled one of them would have been answered as that subcommand. It now skips the global --host/--port options the engine passes and classifies the call by the word that follows, in a pure function covered by unit tests. Any `backends` call other than `install` still lists, so a flag the engine adds later gets the table rather than an empty success. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/src/bin/fake-lemonade.rs | 91 ++++++++++++++++++--- 1 file changed, 81 insertions(+), 10 deletions(-) diff --git a/tests/e2e-cucumber/src/bin/fake-lemonade.rs b/tests/e2e-cucumber/src/bin/fake-lemonade.rs index 40c4382fd3..15d85b95be 100644 --- a/tests/e2e-cucumber/src/bin/fake-lemonade.rs +++ b/tests/e2e-cucumber/src/bin/fake-lemonade.rs @@ -96,25 +96,96 @@ fn run_lemond() -> ExitCode { const LEMOND_MAX_LIFETIME: Duration = Duration::from_mins(10); fn run_lemonade_cli(args: &[String]) -> ExitCode { - if let Some(index) = args.iter().position(|arg| arg == "backends") { - if args.get(index + 1).map(String::as_str) == Some("install") { + match cli_call(args) { + CliCall::BackendsInstall => { // Fail loudly on stderr: the engine passes the child's stderr // through, and the scenario asserts this line to prove the install // it saw fail was this fake's rather than a real runtime's. eprintln!( "{FAKE_LEMONADE_INSTALL_REFUSAL} for @id:serve-lemonade-preparation-recovery" ); - return ExitCode::FAILURE; + ExitCode::FAILURE + } + CliCall::Backends => { + print!("{BACKENDS_TABLE}"); + ExitCode::SUCCESS } - print!("{BACKENDS_TABLE}"); - return ExitCode::SUCCESS; + // `status` succeeds, and so does any other subcommand: the scenario + // never reaches model load, and failing here would mask the failure it + // is actually asserting. + CliCall::Status | CliCall::Other => ExitCode::SUCCESS, } +} + +/// Which of the calls this fake answers `args` is. +#[derive(Debug, PartialEq, Eq)] +enum CliCall { + Status, + Backends, + BackendsInstall, + Other, +} - if args.iter().any(|arg| arg == "status") { - return ExitCode::SUCCESS; +/// Classifies a `lemonade` call by its subcommand: the first word after the +/// global `--host H` and `--port P` options the engine passes ahead of it. +/// Matching the subcommand position, not a word anywhere in `args`, keeps an +/// argument that merely equals `status` or `backends` from being taken as one. +fn cli_call(args: &[String]) -> CliCall { + let mut rest = args; + while let [flag, _value, tail @ ..] = rest { + if flag != "--host" && flag != "--port" { + break; + } + rest = tail; + } + match rest { + [command] if command == "status" => CliCall::Status, + [command, action, ..] if command == "backends" && action == "install" => { + CliCall::BackendsInstall + } + // Any other `backends` call lists, so a flag the engine may add later + // still gets the table rather than an empty success. + [command, ..] if command == "backends" => CliCall::Backends, + _ => CliCall::Other, } +} - // Any other subcommand: succeed quietly. The scenario never reaches model - // load, and failing here would mask the failure it is actually asserting. - ExitCode::SUCCESS +#[cfg(test)] +mod tests { + use super::{CliCall, cli_call}; + + fn call(args: &[&str]) -> CliCall { + cli_call(&args.iter().map(|arg| (*arg).to_owned()).collect::>()) + } + + #[test] + fn the_engine_calls_are_recognised_after_the_global_options() { + assert_eq!( + call(&["--host", "127.0.0.1", "--port", "8000", "status"]), + CliCall::Status + ); + assert_eq!(call(&["backends"]), CliCall::Backends); + assert_eq!(call(&["backends", "--verbose"]), CliCall::Backends); + assert_eq!( + call(&[ + "--host", + "127.0.0.1", + "--port", + "8000", + "backends", + "install", + "llamacpp:rocm", + ]), + CliCall::BackendsInstall + ); + } + + #[test] + fn a_word_outside_the_subcommand_position_is_not_a_subcommand() { + assert_eq!(call(&["pull", "status"]), CliCall::Other); + assert_eq!(call(&["pull", "backends", "install"]), CliCall::Other); + assert_eq!(call(&["--host", "status", "pull"]), CliCall::Other); + assert_eq!(call(&["status", "extra"]), CliCall::Other); + assert_eq!(call(&[]), CliCall::Other); + } } From 6898ef918a7d506f949e5b8e76fa2e543f29b851 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 18:51:54 +0200 Subject: [PATCH 6/9] docs(architecture): sort the runtime_pins row before runtime_root The e2e harness table is alphabetical. Signed-off-by: Roman Inflianskas --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index e8022f0553..8c502ca118 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -806,8 +806,8 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/powershell_text_rule.rs`](../tests/e2e-cucumber/src/powershell_text_rule.rs) | The text rule for the Windows lines `rocm fix` and `rocm diagnose` print, a copy of the rule in `rocm-core` checked against a shared fixture table. | flat | | [`src/quit_and_wait.rs`](../tests/e2e-cucumber/src/quit_and_wait.rs) | Retry loop for quitting a TUI that may open a quit-confirm prompt first. | flat | | [`src/reader_failure.rs`](../tests/e2e-cucumber/src/reader_failure.rs) | Failure state shared between a background terminal reader and its waiters. | flat | -| [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | | [`src/runtime_pins.rs`](../tests/e2e-cucumber/src/runtime_pins.rs) | Reads a pinned runtime version out of the workspace-root [`runtime-deps.toml`](../runtime-deps.toml) as plain data, so a planted runtime keyed by that pin (the Lemonade `env_id`) follows a version bump without importing `rocm-deps`. | flat | +| [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | | [`src/send_until.rs`](../tests/e2e-cucumber/src/send_until.rs) | Retry loop for sending an idempotent terminal input until its effect appears. | flat | | [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail. | flat | | [`src/serve_plan.rs`](../tests/e2e-cucumber/src/serve_plan.rs) | Explains why a `rocm serve` run printed no `resolved model:` line by keeping stderr's refusal alongside stdout. | flat | From 37b509d95ee2631a12eb83d96874af24053d082f Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 14:57:38 +0200 Subject: [PATCH 7/9] refactor(lemonade): derive the runtime env_id from one rocm-deps helper The engine writes `lemonade-embeddable-` into the install manifest, and rocm compares that id with what the engine's Detect reports to decide whether its own install is already done. Each side spelled the shape out, so the two could drift apart and send serve to the real multi-gigabyte install. Both now call rocm_deps::lemonade_env_id. The e2e suite still repeats the shape, since it imports nothing from the workspace; a comment there names the helper it mirrors, and serve-18's refusal count catches a drift. Signed-off-by: Roman Inflianskas --- apps/rocm/src/engines_cmd.rs | 4 +++- crates/rocm-deps/src/lib.rs | 25 +++++++++++++++++++++++++ docs/architecture.md | 2 +- engines/lemonade/src/install.rs | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index dab1629cba..ef02d7c791 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -203,7 +203,7 @@ pub(crate) fn env_root_for_service( /// literal and had drifted several minor versions behind what is installed. fn managed_engine_runtime_id(engine: &str) -> String { match engine { - "lemonade" => format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + "lemonade" => rocm_deps::lemonade_env_id(rocm_deps::LEMONADE_VERSION), _ => "managed-engine-runtime".to_owned(), } } @@ -619,6 +619,8 @@ mod tests { .unwrap_err() .to_string(); assert!(error.contains("no active ROCm runtime is configured")); + // Spelled out rather than via `rocm_deps::lemonade_env_id`, so a change + // to that helper's shape turns this red. assert_eq!( resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index ee95efbf0d..4672899d32 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -42,6 +42,22 @@ pub fn lemonade_download_url(version: &str, archive_name: &str) -> String { ) } +/// The `env_id` a Lemonade embeddable runtime of `version` is recorded under. +/// +/// `version` is the bare release number, as in `LEMONADE_VERSION`, not a `v` +/// tag: the id is compared by exact string, so it is not normalised here. +/// +/// For example `lemonade-embeddable-11.5.1`. The engine writes it into the install +/// manifest it creates, and `rocm` compares it with what the engine's `Detect` +/// reports to decide whether its own install is already done, so the two must +/// agree. The e2e suite repeats the shape when it plants a runtime, because it +/// imports nothing from this workspace; its serve-18 scenario fails if the +/// planted id stops matching. +#[must_use] +pub fn lemonade_env_id(version: &str) -> String { + format!("lemonade-embeddable-{version}") +} + #[cfg(test)] mod tests { use super::*; @@ -86,4 +102,13 @@ mod tests { "https://github.com/lemonade-sdk/lemonade/releases/download/v11.6.0/archive.tar.gz" ); } + + #[test] + fn the_lemonade_env_id_names_the_embeddable_and_its_version() { + assert_eq!(lemonade_env_id("11.5.1"), "lemonade-embeddable-11.5.1"); + assert_eq!( + lemonade_env_id(LEMONADE_VERSION), + format!("lemonade-embeddable-{LEMONADE_VERSION}") + ); + } } diff --git a/docs/architecture.md b/docs/architecture.md index f008a0969d..a8d1eb3fda 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -623,7 +623,7 @@ Pinned versions of the third-party runtimes rocm-cli manages, read from the work | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../crates/rocm-deps/src/lib.rs) | Includes the generated constants and derives Lemonade release names and download URLs from them. | flat | +| [`lib.rs`](../crates/rocm-deps/src/lib.rs) | Includes the generated constants and derives Lemonade release names, download URLs and the install `env_id` from them. | flat | ### `crates/e2e-report` diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index b6617fc215..e651054691 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -183,7 +183,7 @@ where ); } Ok(LemonadeInstallManifest { - env_id: format!("lemonade-embeddable-{version}"), + env_id: rocm_deps::lemonade_env_id(&version), version, runtime_dir, lemond, From b59a9ec8b9ceda528516eeb0145fb420fd7bc5ae Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 14:57:38 +0200 Subject: [PATCH 8/9] test(e2e): read the fake Lemonade fixture once when planting it twice The planted runtime installs the same fixture binary as both `lemond` and `lemonade`. install_fixture_binary_as reads the fixture once and writes every destination from that buffer; install_fixture_binary keeps its single-destination form for the other callers. Also names rocm_deps::lemonade_env_id as the helper the planted env_id mirrors. Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/tests/e2e.rs | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 13d4431240..1f707d1bd0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -754,11 +754,13 @@ impl E2eWorld { let lemonade = runtime_dir.join(e2e_cucumber::fixture_binary::fixture_target_name( "lemonade", )); - install_fixture_binary("fake-lemonade", &lemond); - install_fixture_binary("fake-lemonade", &lemonade); + install_fixture_binary_as("fake-lemonade", &[&lemond, &lemonade]); let version = e2e_cucumber::runtime_pins::pinned_version("lemonade"); let manifest = serde_json::json!({ + // Mirrors `rocm_deps::lemonade_env_id`, which the engine and `rocm` + // both use; the suite cannot import it. A drift sends `serve` to + // the real install, which serve-18's refusal count catches. "env_id": format!("lemonade-embeddable-{version}"), "version": version, "runtime_dir": runtime_dir, @@ -1035,18 +1037,25 @@ pub fn rocmd_binary() -> PathBuf { /// and `rocm-core` is a dev-dependency this crate's library target cannot /// reach, only this test binary. pub fn install_fixture_binary(name: &str, dest: &Path) { - if let Some(parent) = dest.parent() { - std::fs::create_dir_all(parent) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", parent.display())); - } + install_fixture_binary_as(name, &[dest]); +} + +/// [`install_fixture_binary`] under several names, reading the fixture once. +pub fn install_fixture_binary_as(name: &str, dests: &[&Path]) { let contents = std::fs::read(e2e_cucumber::fixture_binary::fixture_binary(name)) .unwrap_or_else(|e| panic!("failed to read the {name} stand-in: {e}")); - rocm_core::fake_executable::write_fake_executable(dest, contents).unwrap_or_else(|e| { - panic!( - "failed to install the {name} stand-in at {}: {e}", - dest.display() - ) - }); + for dest in dests { + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", parent.display())); + } + rocm_core::fake_executable::write_fake_executable(dest, &contents).unwrap_or_else(|e| { + panic!( + "failed to install the {name} stand-in at {}: {e}", + dest.display() + ) + }); + } } /// Spawn the real `rocm` binary with the scenario's isolated env. From e0c4e8dc00a2da410e787d13bc56544d494132dc Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 15:11:51 +0200 Subject: [PATCH 9/9] test(e2e): run serve-18 with --wait so a late engine death fails it The backend install fails inside the spawned managed engine. Without --wait, rocm serve notices an engine that dies only within its 200 ms post-spawn check and reports a later death as `starting` with exit 0 once the 45 s readiness probe gives up. On a loaded host the engine missed that window and serve-18 failed with "serve unexpectedly succeeded". With --wait a later death fails the serve with a pointer to `rocm logs --service `; the step follows that pointer, as a user would, before asserting the retry and recovery lines. An engine that dies inside the window still fails the serve with its log tail. engine_death_report tells the two messages apart and is unit-tested on the CLI's wording; a failed serve carrying neither fails the step and names both, instead of surfacing as a missing retry line. The default (no --wait) behaviour is a product defect and is left for its own fix. Signed-off-by: Roman Inflianskas --- docs/architecture.md | 2 +- .../features/model_serving.feature | 5 +- tests/e2e-cucumber/src/serve_log.rs | 64 ++++++++++++++++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 44 ++++++++++++- 4 files changed, 107 insertions(+), 8 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index a8d1eb3fda..2f5daf3982 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -814,7 +814,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/runtime_pins.rs`](../tests/e2e-cucumber/src/runtime_pins.rs) | Reads a pinned runtime version out of the workspace-root [`runtime-deps.toml`](../runtime-deps.toml) as plain data, so a planted runtime keyed by that pin (the Lemonade `env_id`) follows a version bump without importing `rocm-deps`. | flat | | [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | | [`src/send_until.rs`](../tests/e2e-cucumber/src/send_until.rs) | Retry loop for sending an idempotent terminal input until its effect appears. | flat | -| [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail. | flat | +| [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail; and tells apart the two ways a failed `rocm serve --wait` reports a dead engine (`engine_death_report`). | flat | | [`src/serve_plan.rs`](../tests/e2e-cucumber/src/serve_plan.rs) | Explains why a `rocm serve` run printed no `resolved model:` line by keeping stderr's refusal alongside stdout. | flat | | [`src/shared_runtime.rs`](../tests/e2e-cucumber/src/shared_runtime.rs) | Picks which runtime a scenario activates out of the shared pre-warmed runtime tree, which may hold more than one. | flat | | [`src/stand_in_uname.rs`](../tests/e2e-cucumber/src/stand_in_uname.rs) | The shell script a scenario plants as a stand-in `uname`: `-v` answers a fixed string, `-r` runs a fragment the scenario chooses, and every call is logged so a scenario can count how often the kernel release was asked for. | flat | diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 37d09fc04c..b77bc29be1 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -206,7 +206,10 @@ Feature: Model serving # so the harness also plants one under a simulated host root, which re-roots # Linux paths only; hence Linux. Windows is excluded for a second reason too: # there `rocm serve` does not notice a managed engine dying at startup (#467), - # so the failure would not reach the user at all. + # so the failure would not reach the user at all. The install fails inside the + # spawned engine, so the serve runs with `--wait`: without it, `rocm serve` + # notices only an engine that dies within 200 ms of the spawn and reports a + # later death as `starting` with exit 0. @id:serve-lemonade-preparation-recovery @requires-os:linux Scenario: serve-18 - Repeated Lemonade preparation failure gives the user a recovery path Given Lemonade preparation cannot complete diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index b990259855..7fb8ccdf30 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -360,12 +360,46 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { report } +/// How a managed `rocm serve --wait` reported an engine that died before it +/// became ready, read from its stderr. +#[derive(Debug, PartialEq, Eq)] +pub enum EngineDeathReport { + /// It died inside the post-spawn check, and the serve's error already quotes + /// the tail of its log ("managed engine exited immediately ..."). + LogTailQuoted, + /// It died later, and the serve's error points at + /// `rocm logs --service ` for its output (the readiness wait's message). + LogsPointer(String), + /// Neither: the CLI's wording changed, or the failure was something else. + Unrecognised, +} + +/// Classifies a failed `rocm serve --wait`'s stderr by its engine-death message. +/// +/// The wording matched here is the CLI's: +/// `managed_engine_startup_failure_detail` (the 200 ms post-spawn check) and +/// `died_during_wait` (the readiness wait), both in `apps/rocm/src/service_launch/`. +pub fn engine_death_report(stderr: &str) -> EngineDeathReport { + if stderr.contains("managed engine exited immediately") { + return EngineDeathReport::LogTailQuoted; + } + stderr + .split_once("see `rocm logs --service ") + .and_then(|(_, rest)| rest.split_once('`')) + .map(|(id, _)| id.trim_matches(|c| c == '\'' || c == '"')) + .filter(|id| !id.is_empty()) + .map_or(EngineDeathReport::Unrecognised, |id| { + EngineDeathReport::LogsPointer(id.to_owned()) + }) +} + #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, archive_service_log, - engine_from_serve_args, parse_log_path, read_clamped, read_clamped_file, reported_engine, - serve_attempt_report, serve_cli_budget, service_log_tail, slugify, tail_lines, + EngineDeathReport, MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, + archive_service_log, engine_death_report, engine_from_serve_args, parse_log_path, + read_clamped, read_clamped_file, reported_engine, serve_attempt_report, serve_cli_budget, + service_log_tail, slugify, tail_lines, }; #[test] @@ -715,4 +749,28 @@ managed service launched // A silent stream is a finding, so it is labelled rather than dropped. assert!(report.contains("stderr: (empty)"), "{report}"); } + + #[test] + fn engine_death_reports_are_told_apart_by_the_clis_two_messages() { + // `died_during_wait`'s message, as the CLI prints it. + let waited = "Error: the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service lemonade-qwen3-0-6b-gguf-1794920662779` for its output"; + assert_eq!( + engine_death_report(waited), + EngineDeathReport::LogsPointer("lemonade-qwen3-0-6b-gguf-1794920662779".to_owned()) + ); + // `managed_engine_startup_failure_detail`'s message. + let immediate = "Error: managed engine exited immediately with status exit status: 1; inspect /x/svc.log\n\nrecent startup log output:\nretrying once"; + assert_eq!( + engine_death_report(immediate), + EngineDeathReport::LogTailQuoted + ); + assert_eq!( + engine_death_report("Error: no usable AMD GPU detected"), + EngineDeathReport::Unrecognised + ); + assert_eq!( + engine_death_report("see `rocm logs --service ` for its output"), + EngineDeathReport::Unrecognised + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 1659de78cf..098c95016f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use std::fmt::Write as _; use std::path::PathBuf; use std::sync::LazyLock; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -18,8 +19,8 @@ use e2e_cucumber::device_memory::{ }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, engine_from_serve_args, reported_engine, - serve_attempt_report, service_log_tail, + EngineDeathReport, ServeAttempt, archive_service_log, engine_death_report, + engine_from_serve_args, reported_engine, serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -1408,9 +1409,24 @@ async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) { world.plant_failing_lemonade_runtime(); } +/// `--wait` is load-bearing. The backend install fails inside the managed +/// engine process, after `rocm serve` has spawned it. Without `--wait`, serve +/// only notices an engine that dies within its 200 ms post-spawn check; one +/// that dies later is reported as `starting` with exit 0 once the readiness +/// probe gives up, so on a loaded host this scenario would race that window. +/// With `--wait` a later death fails the serve too, once the 45 s readiness +/// probe ends, so the slow path costs that long. +/// +/// Both ways the failure reaches the user are followed: an engine that died +/// inside the window fails the serve with its log tail, and one that died +/// later fails it with a pointer to `rocm logs --service `, which this step +/// then runs, as the user would. The retry and recovery lines are the engine's +/// own output either way, so the assertions read the serve output together +/// with that log. A failed serve carrying neither message fails here, naming +/// both, rather than as a missing retry line later. #[when("the user serves a model with Lemonade")] async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { - let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + let (mut stdout, stderr, rc) = crate::run_rocm_with_scenario_env( world, &[ "serve", @@ -1418,8 +1434,30 @@ async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { "--engine", "lemonade", "--managed", + "--wait", ], ); + if rc != 0 && !stderr.contains("no usable AMD GPU detected") { + match engine_death_report(&stderr) { + EngineDeathReport::LogTailQuoted => {} + EngineDeathReport::LogsPointer(service_id) => { + let (logs, logs_stderr, logs_rc) = + crate::run_rocm(world, &["logs", "--service", &service_id]); + assert_eq!( + logs_rc, 0, + "`rocm logs --service {service_id}`, which the failed serve pointed at, \ + failed:\n{logs}\n{logs_stderr}" + ); + let _ = write!(stdout, "\n--- rocm logs --service {service_id} ---\n{logs}"); + } + EngineDeathReport::Unrecognised => panic!( + "serve failed without either engine-death message: neither the \ + post-spawn check's \"managed engine exited immediately\" log tail nor \ + the readiness wait's \"see `rocm logs --service `\" pointer. If the \ + CLI reworded one, update `engine_death_report`:\n{stdout}\n{stderr}" + ), + } + } world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); world.cli_rc = Some(rc);