diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 96b63f1b9c..e70daaf9e7 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -379,9 +379,9 @@ jobs: # superset of this one. `xtask/src/workflow_contract.rs` asserts the # floor for that reason. # - # The floor is what matters here. The suite's deterministic failure - # seams (e.g. the scripted Lemonade backend-install failure) are - # compiled out without it, so a pre-built binary that omits the feature + # The floor is what matters here. The suite's test seams (e.g. the + # simulated host root, ROCM_CLI_TEST_HOST_ROOT) are compiled out + # without it, so a pre-built binary that omits the feature # leaves those scenarios unable to reach their premise — they then fail # as regressions on whichever lane happens to select them. Every lane # that pre-builds and exports ROCM_CLI_BINARY must pass it. diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index dab1629cba..ef02d7c791 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -203,7 +203,7 @@ pub(crate) fn env_root_for_service( /// literal and had drifted several minor versions behind what is installed. fn managed_engine_runtime_id(engine: &str) -> String { match engine { - "lemonade" => format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), + "lemonade" => rocm_deps::lemonade_env_id(rocm_deps::LEMONADE_VERSION), _ => "managed-engine-runtime".to_owned(), } } @@ -619,6 +619,8 @@ mod tests { .unwrap_err() .to_string(); assert!(error.contains("no active ROCm runtime is configured")); + // Spelled out rather than via `rocm_deps::lemonade_env_id`, so a change + // to that helper's shape turns this red. assert_eq!( resolve_engine_install_runtime_id(&paths, &RocmCliConfig::default(), "lemonade", None)?, format!("lemonade-embeddable-{}", rocm_deps::LEMONADE_VERSION), diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index f359699db3..8018486029 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -420,13 +420,17 @@ fn aim_board_note( /// policy, on a host the probe *authoritatively* reports as having no usable /// AMD GPU. /// -/// Not "exactly when the bail fires", and the difference is only in test-hook -/// builds: the bail also requires `!scripted_backend_failure` (see the `if` -/// below it), which is `cfg!(feature = "e2e-test-hooks")`/env-gated. Under that -/// scenario the gate can open while the bail cannot fire. That is deliberate — -/// the hook stands in for the GPU the lane does not have, so the reuse path it -/// exercises must still behave as it would on a real GPU-less host — and it -/// costs nothing in a shipped binary, where the term can never be true. +/// Not "exactly when the bail fires". In every build the bail is also skipped +/// when this gate's own answer is a live service to reuse (`reuse_existing`). +/// E2E builds add two more skip terms (see the `if` below it), each env-gated +/// and compiled in only with its feature: `!e2e_simulate_oom_launch()` +/// (`e2e-oom-fault-injection`, either mode) stands in for the GPU the +/// fabricated-launch scenarios' lane does not have, and `!scripted_free_space` (`e2e-test-hooks`) lets the +/// Lemonade free-space refusal, which `rocm serve` only reaches past the bail, +/// be exercised on a GPU-less lane. Under either hook the gate can open while +/// the bail cannot fire. That is deliberate — the pre-gate must still behave as +/// it would on a real GPU-less host — and it costs nothing in a shipped binary, +/// where neither hook term can be true. /// /// `host_has_no_usable_gpu` is therefore the bail's own predicate, computed once /// by the caller and handed to both, so the two cannot drift apart. They did: @@ -861,24 +865,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // // Skipped for cpu_only and when reusing an already-running service (nothing is // launched); permissive when availability cannot be probed on this platform - // (probe returns `None`). The E2E-only - // backend-failure scenario bypasses this host precondition so the black-box - // test reaches Lemonade's backend boundary without real GPU hardware, and the - // E2E-only launch fault injection (either mode) likewise stands in for the - // GPU it does not have. + // (probe returns `None`). The E2E-only launch fault injection (either mode) + // stands in for the GPU its lane does not have. // - // The scripted free-space override opens the same waiver: it exists to refuse a + // The scripted free-space override waives it too: it exists to refuse a // Lemonade install on a drive that cannot hold it, which `rocm serve` only // reaches past this bail. - let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") - && [ - "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", - "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", - ] - .iter() - .any(|variable| std::env::var_os(variable).is_some()); + let scripted_free_space = cfg!(feature = "e2e-test-hooks") + && std::env::var_os("ROCM_E2E_LEMONADE_FREE_SPACE_BYTES").is_some(); if !cpu_only - && !scripted_backend_failure + && !scripted_free_space && !e2e_simulate_oom_launch() && !reuse_existing && host_has_no_usable_gpu @@ -892,10 +888,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted - // backend failure is the one remaining term. Handed to the spawn, which + // free-space override is the one remaining term. Handed to the spawn, which // re-checks reuse authoritatively under the launch lock and raises the // refusal itself if the service is gone by then. - let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_backend_failure; + let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_free_space; // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via // `HIP_VISIBLE_DEVICES`; those orderings can diverge when // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. diff --git a/crates/rocm-deps/src/lib.rs b/crates/rocm-deps/src/lib.rs index ee95efbf0d..4672899d32 100644 --- a/crates/rocm-deps/src/lib.rs +++ b/crates/rocm-deps/src/lib.rs @@ -42,6 +42,22 @@ pub fn lemonade_download_url(version: &str, archive_name: &str) -> String { ) } +/// The `env_id` a Lemonade embeddable runtime of `version` is recorded under. +/// +/// `version` is the bare release number, as in `LEMONADE_VERSION`, not a `v` +/// tag: the id is compared by exact string, so it is not normalised here. +/// +/// For example `lemonade-embeddable-11.5.1`. The engine writes it into the install +/// manifest it creates, and `rocm` compares it with what the engine's `Detect` +/// reports to decide whether its own install is already done, so the two must +/// agree. The e2e suite repeats the shape when it plants a runtime, because it +/// imports nothing from this workspace; its serve-18 scenario fails if the +/// planted id stops matching. +#[must_use] +pub fn lemonade_env_id(version: &str) -> String { + format!("lemonade-embeddable-{version}") +} + #[cfg(test)] mod tests { use super::*; @@ -86,4 +102,13 @@ mod tests { "https://github.com/lemonade-sdk/lemonade/releases/download/v11.6.0/archive.tar.gz" ); } + + #[test] + fn the_lemonade_env_id_names_the_embeddable_and_its_version() { + assert_eq!(lemonade_env_id("11.5.1"), "lemonade-embeddable-11.5.1"); + assert_eq!( + lemonade_env_id(LEMONADE_VERSION), + format!("lemonade-embeddable-{LEMONADE_VERSION}") + ); + } } diff --git a/docs/architecture.md b/docs/architecture.md index 75d87bf0ec..2f5daf3982 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -623,7 +623,7 @@ Pinned versions of the third-party runtimes rocm-cli manages, read from the work | File | Responsibility | Class | | --- | --- | --- | -| [`lib.rs`](../crates/rocm-deps/src/lib.rs) | Includes the generated constants and derives Lemonade release names and download URLs from them. | flat | +| [`lib.rs`](../crates/rocm-deps/src/lib.rs) | Includes the generated constants and derives Lemonade release names, download URLs and the install `env_id` from them. | flat | ### `crates/e2e-report` @@ -811,9 +811,10 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`src/powershell_text_rule.rs`](../tests/e2e-cucumber/src/powershell_text_rule.rs) | The text rule for the Windows lines `rocm fix` and `rocm diagnose` print, a copy of the rule in `rocm-core` checked against a shared fixture table. | flat | | [`src/quit_and_wait.rs`](../tests/e2e-cucumber/src/quit_and_wait.rs) | Retry loop for quitting a TUI that may open a quit-confirm prompt first. | flat | | [`src/reader_failure.rs`](../tests/e2e-cucumber/src/reader_failure.rs) | Failure state shared between a background terminal reader and its waiters. | flat | +| [`src/runtime_pins.rs`](../tests/e2e-cucumber/src/runtime_pins.rs) | Reads a pinned runtime version out of the workspace-root [`runtime-deps.toml`](../runtime-deps.toml) as plain data, so a planted runtime keyed by that pin (the Lemonade `env_id`) follows a version bump without importing `rocm-deps`. | flat | | [`src/runtime_root.rs`](../tests/e2e-cucumber/src/runtime_root.rs) | `check_runtime_root_not_nested`: judges the active runtime folder `rocm examine` prints and rejects a doubly nested `runtimes/wheel` path. | flat | | [`src/send_until.rs`](../tests/e2e-cucumber/src/send_until.rs) | Retry loop for sending an idempotent terminal input until its effect appears. | flat | -| [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail. | flat | +| [`src/serve_log.rs`](../tests/e2e-cucumber/src/serve_log.rs) | Reads back a managed serve's service log and builds the failure report for a serve step, quoting the log tail; and tells apart the two ways a failed `rocm serve --wait` reports a dead engine (`engine_death_report`). | flat | | [`src/serve_plan.rs`](../tests/e2e-cucumber/src/serve_plan.rs) | Explains why a `rocm serve` run printed no `resolved model:` line by keeping stderr's refusal alongside stdout. | flat | | [`src/shared_runtime.rs`](../tests/e2e-cucumber/src/shared_runtime.rs) | Picks which runtime a scenario activates out of the shared pre-warmed runtime tree, which may hold more than one. | flat | | [`src/stand_in_uname.rs`](../tests/e2e-cucumber/src/stand_in_uname.rs) | The shell script a scenario plants as a stand-in `uname`: `-v` answers a fixed string, `-r` runs a fragment the scenario chooses, and every call is logged so a scenario can count how often the kernel release was asked for. | flat | @@ -827,6 +828,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | File | Responsibility | Class | | --- | --- | --- | | [`src/bin/fake-docker.rs`](../tests/e2e-cucumber/src/bin/fake-docker.rs) | A `docker` stand-in that always fails, optionally logging its arguments, for the AIM Docker-unavailable scenarios. | flat | +| [`src/bin/fake-lemonade.rs`](../tests/e2e-cucumber/src/bin/fake-lemonade.rs) | The `lemonade` CLI and `lemond` server of a planted Lemonade runtime, chosen by `argv[0]`: `lemond` stays alive for a bounded time, and `lemonade backends install` fails, for `@id:serve-lemonade-preparation-recovery`. | flat | | [`src/bin/fake-python.rs`](../tests/e2e-cucumber/src/bin/fake-python.rs) | A stand-in for the runtime's Python that answers the ComfyUI install's torch-stack probe and post-install GPU check. | flat | | [`src/bin/fake-tailscale.rs`](../tests/e2e-cucumber/src/bin/fake-tailscale.rs) | A Tailscale CLI stand-in that serves the status document named by `FAKE_TAILSCALE_STATUS`, for the `rocm remote` scenarios. | flat | | [`src/bin/fake-uv.rs`](../tests/e2e-cucumber/src/bin/fake-uv.rs) | A `uv` stand-in whose output, exit code, argument log and stdin handling are set by `FAKE_UV_*` variables, for the ComfyUI scenarios. | flat | diff --git a/docs/testing.md b/docs/testing.md index 9c42bb5fd1..8f94b3f75b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -992,8 +992,9 @@ pushed (`PRE_COMMIT_FROM_REF..PRE_COMMIT_TO_REF`, the commits the remote does no The directory-based selection cannot see a crate's tests reading a file from outside the crate, so the hook widens it: any Rust file also selects `xtask` (its contract tests scan every source file), `README.md`, `docs/` and `skills/` also select `rocm` and `xtask` (the advised-commands -test reads them and `rocm` compiles in the assistant skill), and the e2e PowerShell fixture also -selects `rocm-core` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in +test reads them and `rocm` compiles in the assistant skill), and the e2e `fixtures/` directory also +selects `rocm-core` (its tests `include_str!` the PowerShell cases), and the fake Lemonade runtime's +backends table there also selects `rocm-engine-lemonade` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in [prepush.rs](../xtask/src/prepush.rs), and a unit test scans the workspace for `include_str!`, `include_bytes!` and manifest-relative reads that leave their crate and fails when one is not covered, so a new cross-crate read cannot be added without a decision. The scan has a limit: a read @@ -2049,7 +2050,7 @@ Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES`, compiled only into the `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the disk that nothing was downloaded. That variable also waives serve's no-GPU - refusal, as serve-18's failure hook does. + refusal, so the scenario reaches the install on a host with no GPU. - `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime and checks `rocm storage report` lists both and flags the unused one. - `lemonade-installs-04` uninstalls that runtime and checks the install record diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index b6617fc215..e651054691 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -183,7 +183,7 @@ where ); } Ok(LemonadeInstallManifest { - env_id: format!("lemonade-embeddable-{version}"), + env_id: rocm_deps::lemonade_env_id(&version), version, runtime_dir, lemond, diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 05daff4add..990af0363b 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -57,8 +57,6 @@ pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; pub(crate) const DEFAULT_MODEL: &str = "Qwen3-4B-Instruct-2507-GGUF"; pub(crate) const LLAMACPP_RECIPE: &str = "llamacpp"; pub(crate) const ROCM_BACKEND_NAME: &str = "rocm"; -#[cfg(feature = "e2e-test-hooks")] -const BACKEND_INSTALL_FAILURE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE"; const DEFAULT_LOG_TAIL_LINES: usize = 200; /// How long a stop waits for the server to actually exit after each signal /// before reporting a timeout (or, under `force`, escalating to `SIGKILL`). @@ -398,16 +396,6 @@ fn install_and_record( fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; - // Debug builds expose a deterministic failure seam for the black-box CLI - // scenario that pins retry count and terminal recovery guidance. Keep it at - // the backend phase: the real defect happens after the embeddable is ready, - // and exercising it must not download or alter a runtime on the test host. - #[cfg(feature = "e2e-test-hooks")] - if std::env::var_os(BACKEND_INSTALL_FAILURE_TEST_ENV).is_some() { - crate::llamacpp_backend::install_llamacpp_backend_with_retry(|| { - anyhow::bail!("scripted Lemonade backend install failure") - })?; - } let env_root = request .env_root .as_deref() diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 48a9caf91b..20710c8b44 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -394,6 +394,36 @@ mod tests { } } + /// The E2E fake runtime's `lemonade backends` table must drive this engine + /// to attempt a `llamacpp:rocm` install: that attempt is the whole premise + /// of `@id:serve-lemonade-preparation-recovery`, which then fails it. + /// + /// The fixture prints the table from a data file, and this test reads the + /// same file, so editing it there is checked here, on every lane that runs + /// unit tests. A parser change that stopped accepting it would otherwise + /// make the scenario stop reaching the failure it asserts, with no unit + /// test noticing. + #[test] + fn the_e2e_fake_runtime_table_drives_a_rocm_backend_install() { + const TABLE: &str = + include_str!("../../../tests/e2e-cucumber/fixtures/lemonade_backends_table.txt"); + let backends = parse_llamacpp_backend_statuses(TABLE); + assert_eq!( + backends, + vec![ + ("rocm".to_owned(), "installable".to_owned()), + ("vulkan".to_owned(), "unsupported".to_owned()), + ] + ); + // `false` = not already installed, which is what makes the engine run + // the install the fixture then refuses. `installed` would skip it, and + // the scenario would never see a failure. + assert_eq!( + select_best_llamacpp_backend(&backends), + Some(("rocm".to_owned(), false)) + ); + } + #[test] fn parses_llamacpp_backends_from_table() { let output = "\ diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 3efc130f5d..ce25d46c18 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -40,6 +40,13 @@ path = "src/bin/fake-python.rs" name = "fake-docker" path = "src/bin/fake-docker.rs" +# Stand-in for a Lemonade runtime's `lemonade` and `lemond` executables, planted +# by the serve-preparation-recovery scenario so Lemonade's backend install fails +# at a real child process rather than at a seam compiled into the engine. +[[bin]] +name = "fake-lemonade" +path = "src/bin/fake-lemonade.rs" + [dependencies] axum.workspace = true cucumber = { version = "0.23", features = ["output-json", "output-junit"] } diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d871ab21d1..f10498f733 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -253,11 +253,13 @@ The `.feature` file is both the spec and the test input — cucumber reads it at [`tests/e2e.rs`](tests/e2e.rs) plants it (it needs the `rocm-core` dev-dependency the library target can't reach). See [`fake-uv`](src/bin/fake-uv.rs), [`fake-python`](src/bin/fake-python.rs), - and [`fake-tailscale`](src/bin/fake-tailscale.rs) for the pattern. + [`fake-tailscale`](src/bin/fake-tailscale.rs), and + [`fake-lemonade`](src/bin/fake-lemonade.rs) (one binary planted under two + names, taking its role from `argv[0]`) for the pattern. ## Design principles -- **Black-box only.** Step functions interact with `rocm` and explicitly configured `rocmd` through their CLIs and HTTP endpoints. No imports from the rocm-cli codebase. Where a scenario needs the CLI to know about a running server, it plants a managed-service record as plain JSON matching the on-disk schema (see `register_mock_service`) — the same file `rocm serve --managed` would write — rather than importing the record type. (A harness *unit* test may read a planted record back through `rocm_core::ManagedServiceRecord`, to tie the writer's field names to the ones the CLI reads; steps do not.) +- **Black-box only.** Step functions interact with `rocm` and explicitly configured `rocmd` through their CLIs and HTTP endpoints. No imports from the rocm-cli codebase. Where a scenario needs the CLI to know about a running server, it plants a managed-service record as plain JSON matching the on-disk schema (see `register_mock_service`) — the same file `rocm serve --managed` would write — rather than importing the record type. (A harness *unit* test may read a planted record back through `rocm_core::ManagedServiceRecord`, to tie the writer's field names to the ones the CLI reads; steps do not.) Where a scenario plants something keyed by a pinned runtime version, such as the Lemonade runtime's `env_id`, it reads the version from the workspace pin file [`runtime-deps.toml`](../../runtime-deps.toml) as data (see [`runtime_pins`](src/runtime_pins.rs)) rather than importing `rocm-deps`. - **Isolated state.** Each scenario uses isolated config, data, and cache directories. Tests never touch `~/.rocm`. - **Behavioral language.** Feature files describe what users care about, not implementation details. How steps are implemented (mock vs real, which port, which API) stays in the step functions. - **A scenario about a sequence of chat completions scripts them.** `MockServer::chat_requests` records every chat completion request body even with the fixed stub reply, so a scenario can count completions and read each prompt. A script makes a double ask visible in what the user sees as well: `MockServer::start_with_chat_script` hands out one distinct reply per chat completion request (on either the `/v1/chat/completions` or the unversioned `/chat/completions` route), so the plan that gets printed or returned shows which call produced it, and a request past the end of the script gets HTTP 500 (and is still recorded), so asking more often than scripted fails loudly instead of recycling a reply. Plant the service with `ServiceRecordOptions { inference_verified: true, .. }` so the CLI's own readiness probe does not consume the first scripted reply. `planner_steps.rs` is the worked example. diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature index e55ad7fe16..24d0badd39 100644 --- a/tests/e2e-cucumber/features/lemonade_installs.feature +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -27,7 +27,7 @@ Feature: Lemonade installs on disk # `rocm serve` reaches the same install on its own, which is how the second # copy used to appear without a word. The free-space override also stands in - # for the GPU the no-GPU lane does not have, as serve-18's failure hook does. + # for the GPU the no-GPU lane does not have. @id:lemonade-installs-serve-refused-without-space Scenario: lemonade-installs-02 - Serving with Lemonade on a drive that cannot hold it is refused before downloading Given the drive for a new Lemonade install cannot hold even Lemonade itself diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 096bd0825c..b77bc29be1 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -197,12 +197,20 @@ Feature: Model serving Then serving is refused before any engine starts And the user is told the two selectors cannot be combined - # The failure is injected at Lemonade's backend-install boundary in debug/test - # builds, after the CLI has selected Lemonade but before any runtime download or - # machine mutation. That makes the user-visible retry and final recovery command - # deterministic on the blocking no-GPU lane rather than relying on a real 3 GiB - # transfer to fail at just the right moment. - @id:serve-lemonade-preparation-recovery @requires-no-gpu + # The harness plants a Lemonade runtime whose `lemonade backends install` + # always exits non-zero, so the failure happens at a real boundary -- a + # non-zero child process -- after the CLI has selected Lemonade but before any + # runtime download or machine mutation, and the retry and recovery command are + # deterministic without a real multi-gigabyte transfer failing at the right + # moment. `rocm serve` refuses before preparation on a host with no usable GPU, + # so the harness also plants one under a simulated host root, which re-roots + # Linux paths only; hence Linux. Windows is excluded for a second reason too: + # there `rocm serve` does not notice a managed engine dying at startup (#467), + # so the failure would not reach the user at all. The install fails inside the + # spawned engine, so the serve runs with `--wait`: without it, `rocm serve` + # notices only an engine that dies within 200 ms of the spawn and reports a + # later death as `starting` with exit 0. + @id:serve-lemonade-preparation-recovery @requires-os:linux Scenario: serve-18 - Repeated Lemonade preparation failure gives the user a recovery path Given Lemonade preparation cannot complete When the user serves a model with Lemonade diff --git a/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt b/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt new file mode 100644 index 0000000000..44410af293 --- /dev/null +++ b/tests/e2e-cucumber/fixtures/lemonade_backends_table.txt @@ -0,0 +1,4 @@ +Recipe Backend Status +-------- -------- ----------- +llamacpp rocm installable +llamacpp vulkan unsupported diff --git a/tests/e2e-cucumber/src/bin/fake-lemonade.rs b/tests/e2e-cucumber/src/bin/fake-lemonade.rs new file mode 100644 index 0000000000..15d85b95be --- /dev/null +++ b/tests/e2e-cucumber/src/bin/fake-lemonade.rs @@ -0,0 +1,191 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in for the two executables a Lemonade runtime installs: the `lemonade` +//! CLI and the `lemond` server. +//! +//! `@id:serve-lemonade-preparation-recovery` needs Lemonade's llama.cpp backend +//! install to fail deterministically, twice, so the scenario can pin the retry +//! count and the terminal recovery guidance. That failure used to be a +//! `#[cfg(feature = "e2e-test-hooks")]` branch compiled into the Lemonade engine +//! itself, plus a second one in `rocm serve` that waived its no-GPU pre-flight. +//! Driving it from here keeps both out of the binary under test: the CLI spawns +//! these as real subprocesses over the interface it uses for a real runtime, and +//! only the runtime is fake. +//! +//! The harness copies this one binary into a planted runtime directory under +//! both names, so the role is taken from `argv[0]`: +//! +//! - `lemond` -- the server `rocm serve` spawns. It only has to stay alive; the +//! CLI's readiness check polls the `lemonade` CLI, not this process. +//! - `lemonade` -- the CLI the engine shells out to. Three calls matter, and the +//! real binary's contract for each is what this reproduces: +//! - `--host H --port P status` -> exit 0, which is how the engine decides the +//! server came up. +//! - `backends` -> [`BACKENDS_TABLE`], the recipe/backend/status table the +//! engine scrapes. Reporting `llamacpp rocm` as `installable` (not +//! `installed`) is what makes the engine attempt an install rather than +//! reuse one. +//! - `--host H --port P backends install llamacpp:rocm` -> non-zero, the +//! failure the scenario is about. + +use std::path::Path; +use std::process::ExitCode; +use std::time::Duration; + +use e2e_cucumber::fixture_binary::FAKE_LEMONADE_INSTALL_REFUSAL; + +/// What `lemonade backends` prints here. +/// +/// A data file rather than a literal so the Lemonade engine's own unit tests +/// can `include_str!` the same file and check that the engine parses it into a +/// `llamacpp:rocm` install attempt. +const BACKENDS_TABLE: &str = include_str!("../../fixtures/lemonade_backends_table.txt"); + +fn main() -> ExitCode { + // Copied rather than symlinked (self-hosted Windows runners commonly lack + // SeCreateSymbolicLinkPrivilege), so the file name is the only role signal. + let argv0 = std::env::args_os().next().unwrap_or_default(); + let role = Path::new(&argv0) + .file_stem() + .map(|stem| stem.to_string_lossy().into_owned()) + .unwrap_or_default(); + let args: Vec = std::env::args().skip(1).collect(); + + match role.as_str() { + "lemond" => run_lemond(), + "lemonade" => run_lemonade_cli(&args), + other => { + eprintln!( + "fake-lemonade: copied to an unexpected name {other:?}; expected `lemond` or \ + `lemonade`" + ); + ExitCode::FAILURE + } + } +} + +/// Stay alive for as long as the process that spawned us does. +/// +/// Exiting while the engine still runs would turn the scenario's expected +/// backend-install failure into a spurious "server exited" diagnosis. But the +/// engine does not stop `lemond` when the backend install fails (#729): it +/// exits and leaves the server running, and only the scenario's teardown +/// (`stop_managed_services`, best-effort) stops it. So this fake also exits on +/// its own once its parent is gone -- on Unix, when it has been re-parented -- +/// and in any case after [`LEMOND_MAX_LIFETIME`], so a run whose teardown did +/// not get that far still leaves no process behind. +fn run_lemond() -> ExitCode { + #[cfg(unix)] + let spawned_by = std::os::unix::process::parent_id(); + let started = std::time::Instant::now(); + while started.elapsed() < LEMOND_MAX_LIFETIME { + std::thread::sleep(Duration::from_millis(500)); + #[cfg(unix)] + if std::os::unix::process::parent_id() != spawned_by { + break; + } + } + ExitCode::SUCCESS +} + +/// Backstop on the fake server's lifetime, for a parent that never exits or a +/// platform where re-parenting is not observable. The scenario finishes in +/// seconds, so this is far longer than any run needs. +const LEMOND_MAX_LIFETIME: Duration = Duration::from_mins(10); + +fn run_lemonade_cli(args: &[String]) -> ExitCode { + match cli_call(args) { + CliCall::BackendsInstall => { + // Fail loudly on stderr: the engine passes the child's stderr + // through, and the scenario asserts this line to prove the install + // it saw fail was this fake's rather than a real runtime's. + eprintln!( + "{FAKE_LEMONADE_INSTALL_REFUSAL} for @id:serve-lemonade-preparation-recovery" + ); + ExitCode::FAILURE + } + CliCall::Backends => { + print!("{BACKENDS_TABLE}"); + ExitCode::SUCCESS + } + // `status` succeeds, and so does any other subcommand: the scenario + // never reaches model load, and failing here would mask the failure it + // is actually asserting. + CliCall::Status | CliCall::Other => ExitCode::SUCCESS, + } +} + +/// Which of the calls this fake answers `args` is. +#[derive(Debug, PartialEq, Eq)] +enum CliCall { + Status, + Backends, + BackendsInstall, + Other, +} + +/// Classifies a `lemonade` call by its subcommand: the first word after the +/// global `--host H` and `--port P` options the engine passes ahead of it. +/// Matching the subcommand position, not a word anywhere in `args`, keeps an +/// argument that merely equals `status` or `backends` from being taken as one. +fn cli_call(args: &[String]) -> CliCall { + let mut rest = args; + while let [flag, _value, tail @ ..] = rest { + if flag != "--host" && flag != "--port" { + break; + } + rest = tail; + } + match rest { + [command] if command == "status" => CliCall::Status, + [command, action, ..] if command == "backends" && action == "install" => { + CliCall::BackendsInstall + } + // Any other `backends` call lists, so a flag the engine may add later + // still gets the table rather than an empty success. + [command, ..] if command == "backends" => CliCall::Backends, + _ => CliCall::Other, + } +} + +#[cfg(test)] +mod tests { + use super::{CliCall, cli_call}; + + fn call(args: &[&str]) -> CliCall { + cli_call(&args.iter().map(|arg| (*arg).to_owned()).collect::>()) + } + + #[test] + fn the_engine_calls_are_recognised_after_the_global_options() { + assert_eq!( + call(&["--host", "127.0.0.1", "--port", "8000", "status"]), + CliCall::Status + ); + assert_eq!(call(&["backends"]), CliCall::Backends); + assert_eq!(call(&["backends", "--verbose"]), CliCall::Backends); + assert_eq!( + call(&[ + "--host", + "127.0.0.1", + "--port", + "8000", + "backends", + "install", + "llamacpp:rocm", + ]), + CliCall::BackendsInstall + ); + } + + #[test] + fn a_word_outside_the_subcommand_position_is_not_a_subcommand() { + assert_eq!(call(&["pull", "status"]), CliCall::Other); + assert_eq!(call(&["pull", "backends", "install"]), CliCall::Other); + assert_eq!(call(&["--host", "status", "pull"]), CliCall::Other); + assert_eq!(call(&["status", "extra"]), CliCall::Other); + assert_eq!(call(&[]), CliCall::Other); + } +} diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 6771d09dcd..603f5086d7 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1557,6 +1557,51 @@ serve_timeout_secs = 90 } } + /// `serve-18`'s gate decides which lanes exercise Lemonade's preparation + /// retry, so it is pinned against the REAL tag line: a tag edit must fail a + /// test rather than quietly move the scenario. + /// + /// It runs on every Linux lane, GPU or not -- the harness plants the GPU the + /// pre-flight needs under a simulated host root -- which keeps the cadence + /// the old compiled-in seam gave it on the GPU-less every-PR lane. It must + /// not run on Windows, where the host root re-roots nothing and `rocm serve` + /// does not notice a managed engine dying at startup (#467). + #[test] + fn the_lemonade_recovery_scenario_runs_on_every_linux_lane_and_not_on_windows() { + let feature = include_str!("../features/model_serving.feature"); + let m = Expectations::default(); + let decl = ScenarioDecl::from_tags(&feature_tags( + feature, + "serve-lemonade-preparation-recovery", + )); + let gpu_less_linux = HostCapability { + gfx_target: None, + has_amd_gpu: false, + amd_gpu_count: Some(0), + platform_slug: "mock".into(), + ..cap("strix-ubuntu") + }; + for (host, capability) in [ + ("GPU-less Linux", gpu_less_linux), + ("strix-ubuntu", cap("strix-ubuntu")), + ("mi300x", cap("mi300x")), + ("wsl2", cap("wsl2")), + ] { + assert_eq!( + resolve(&decl, &capability, &m, Included::default()), + Expectation::ExpectPass, + "{host} must run serve-18" + ); + } + assert!( + matches!( + resolve(&decl, &cap("strix-windows"), &m, Included::default()), + Expectation::Skip { .. } + ), + "Windows must skip serve-18" + ); + } + /// The single-GPU skip must name the count, not just repeat the `@requires-gpu` /// reason: on Strix Halo a GPU *is* present, so "none detected" would send a /// reader of the report looking for a missing device that is right there. diff --git a/tests/e2e-cucumber/src/fixture_binary.rs b/tests/e2e-cucumber/src/fixture_binary.rs index fa055fa28c..cb90028201 100644 --- a/tests/e2e-cucumber/src/fixture_binary.rs +++ b/tests/e2e-cucumber/src/fixture_binary.rs @@ -19,6 +19,16 @@ use std::path::PathBuf; +/// What the `fake-lemonade` stand-in prints on stderr each time it refuses a +/// `lemonade backends install`. +/// +/// The Lemonade recovery scenario asserts it in `rocm serve`'s output, which +/// proves the install the CLI attempted was the planted runtime's. If the +/// planted manifest stopped matching what `Detect` expects, `serve` would run +/// the real runtime install instead, and this line would be missing. +pub const FAKE_LEMONADE_INSTALL_REFUSAL: &str = + "fake-lemonade: refusing to install a backend; this runtime was planted by the E2E harness"; + /// The path cargo built the `[[bin]]` named `name` at, alongside this test /// binary. pub fn fixture_binary(name: &str) -> PathBuf { diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 29e3345836..27193c9fb4 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -27,6 +27,7 @@ pub mod polled_deadline; pub mod powershell_text_rule; pub mod quit_and_wait; pub mod reader_failure; +pub mod runtime_pins; pub mod runtime_root; pub mod send_until; pub mod serve_log; diff --git a/tests/e2e-cucumber/src/runtime_pins.rs b/tests/e2e-cucumber/src/runtime_pins.rs new file mode 100644 index 0000000000..da0ab5d679 --- /dev/null +++ b/tests/e2e-cucumber/src/runtime_pins.rs @@ -0,0 +1,88 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The pinned third-party runtime versions, read from the workspace-root +//! `runtime-deps.toml` as plain data. +//! +//! That file is the single source of truth for the pins: the product turns it +//! into constants at build time, in the `rocm-deps` crate. The suite is +//! black-box and imports nothing from the rocm-cli codebase (see the README's +//! design principles), so it reads the same file instead. A scenario that must +//! plant something keyed by a pinned version, such as the Lemonade runtime's +//! `env_id`, then follows a version bump with no copy to keep in step. + +use std::path::{Path, PathBuf}; + +/// The pin file, relative to this crate's manifest directory. +const PINS_FILE: &str = "../../runtime-deps.toml"; + +/// Path of the workspace-root pin file. +pub fn pins_path() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join(PINS_FILE) +} + +/// The `version` pinned for `[runtime.]` in the workspace pin file. +/// +/// Panics if the file cannot be read or has no such pin: a scenario that plants +/// a runtime under a missing or misspelled pin must fail loudly, not plant one +/// the CLI does not recognise. +pub fn pinned_version(runtime: &str) -> String { + let path = pins_path(); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())); + version_in(&text, runtime).unwrap_or_else(|e| panic!("{}: {e}", path.display())) +} + +/// The `version` field of `[runtime.]` in a pin file's text. +pub fn version_in(text: &str, runtime: &str) -> Result { + let doc: toml::Table = toml::from_str(text).map_err(|e| format!("not valid TOML: {e}"))?; + doc.get("runtime") + .and_then(toml::Value::as_table) + .and_then(|runtimes| runtimes.get(runtime)) + .and_then(|entry| entry.get("version")) + .and_then(toml::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| format!("no string `version` under [runtime.{runtime}]")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_lemonade_pin_from_the_workspace_file() { + let version = pinned_version("lemonade"); + // The pin is the value the file holds under `[runtime.lemonade]`, read + // independently of `version_in`, so a parser that ignored the runtime + // name or returned a constant would fail here. + let text = std::fs::read_to_string(pins_path()).expect("pin file is readable"); + let in_section = text + .split("[runtime.lemonade]") + .nth(1) + .expect("pin file has a [runtime.lemonade] table"); + let line = in_section + .lines() + .find(|line| line.trim_start().starts_with("version")) + .expect("[runtime.lemonade] has a version line"); + assert!( + line.contains(&format!("\"{version}\"")), + "pinned_version returned {version:?}, the file says {line:?}" + ); + // Same shape the product's own pin check requires: a bare version. + assert!( + !version.is_empty() && !version.starts_with('v'), + "pin must be a bare version, got {version:?}" + ); + } + + #[test] + fn picks_the_named_runtime_and_rejects_a_missing_one() { + let text = + "[runtime.lemonade]\nversion = \"1.2.3\"\n\n[runtime.other]\nversion = \"9.9.9\"\n"; + assert_eq!(version_in(text, "lemonade").as_deref(), Ok("1.2.3")); + assert_eq!(version_in(text, "other").as_deref(), Ok("9.9.9")); + assert!(version_in(text, "absent").is_err()); + assert!(version_in("[runtime.lemonade]\n", "lemonade").is_err()); + } +} diff --git a/tests/e2e-cucumber/src/serve_log.rs b/tests/e2e-cucumber/src/serve_log.rs index b990259855..7fb8ccdf30 100644 --- a/tests/e2e-cucumber/src/serve_log.rs +++ b/tests/e2e-cucumber/src/serve_log.rs @@ -360,12 +360,46 @@ pub fn serve_attempt_report(attempt: &ServeAttempt<'_>) -> String { report } +/// How a managed `rocm serve --wait` reported an engine that died before it +/// became ready, read from its stderr. +#[derive(Debug, PartialEq, Eq)] +pub enum EngineDeathReport { + /// It died inside the post-spawn check, and the serve's error already quotes + /// the tail of its log ("managed engine exited immediately ..."). + LogTailQuoted, + /// It died later, and the serve's error points at + /// `rocm logs --service ` for its output (the readiness wait's message). + LogsPointer(String), + /// Neither: the CLI's wording changed, or the failure was something else. + Unrecognised, +} + +/// Classifies a failed `rocm serve --wait`'s stderr by its engine-death message. +/// +/// The wording matched here is the CLI's: +/// `managed_engine_startup_failure_detail` (the 200 ms post-spawn check) and +/// `died_during_wait` (the readiness wait), both in `apps/rocm/src/service_launch/`. +pub fn engine_death_report(stderr: &str) -> EngineDeathReport { + if stderr.contains("managed engine exited immediately") { + return EngineDeathReport::LogTailQuoted; + } + stderr + .split_once("see `rocm logs --service ") + .and_then(|(_, rest)| rest.split_once('`')) + .map(|(id, _)| id.trim_matches(|c| c == '\'' || c == '"')) + .filter(|id| !id.is_empty()) + .map_or(EngineDeathReport::Unrecognised, |id| { + EngineDeathReport::LogsPointer(id.to_owned()) + }) +} + #[cfg(test)] mod tests { use super::{ - MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, archive_service_log, - engine_from_serve_args, parse_log_path, read_clamped, read_clamped_file, reported_engine, - serve_attempt_report, serve_cli_budget, service_log_tail, slugify, tail_lines, + EngineDeathReport, MAX_ARCHIVED_LOG_BYTES, SERVE_CLI_MARGIN_SECS, ServeAttempt, + archive_service_log, engine_death_report, engine_from_serve_args, parse_log_path, + read_clamped, read_clamped_file, reported_engine, serve_attempt_report, serve_cli_budget, + service_log_tail, slugify, tail_lines, }; #[test] @@ -715,4 +749,28 @@ managed service launched // A silent stream is a finding, so it is labelled rather than dropped. assert!(report.contains("stderr: (empty)"), "{report}"); } + + #[test] + fn engine_death_reports_are_told_apart_by_the_clis_two_messages() { + // `died_during_wait`'s message, as the CLI prints it. + let waited = "Error: the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service lemonade-qwen3-0-6b-gguf-1794920662779` for its output"; + assert_eq!( + engine_death_report(waited), + EngineDeathReport::LogsPointer("lemonade-qwen3-0-6b-gguf-1794920662779".to_owned()) + ); + // `managed_engine_startup_failure_detail`'s message. + let immediate = "Error: managed engine exited immediately with status exit status: 1; inspect /x/svc.log\n\nrecent startup log output:\nretrying once"; + assert_eq!( + engine_death_report(immediate), + EngineDeathReport::LogTailQuoted + ); + assert_eq!( + engine_death_report("Error: no usable AMD GPU detected"), + EngineDeathReport::Unrecognised + ); + assert_eq!( + engine_death_report("see `rocm logs --service ` for its output"), + EngineDeathReport::Unrecognised + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index ae41672ba7..1f707d1bd0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -694,6 +694,110 @@ impl E2eWorld { self.legacy_rocm_path = Some(rocm); } + /// Plant a Lemonade runtime whose llama.cpp backend install always fails, on + /// a simulated host with one AMD GPU, so + /// `@id:serve-lemonade-preparation-recovery` can pin the retry count and the + /// terminal recovery guidance without a real GPU or a multi-gigabyte + /// runtime download. + /// + /// This replaces two `#[cfg(feature = "e2e-test-hooks")]` branches: one in + /// the Lemonade engine that failed the install on an environment variable, + /// and one in `rocm serve` that waived its no-GPU pre-flight for the same + /// variable. The failure now happens at the boundary a broken install fails + /// at -- a non-zero `lemonade backends install` -- and the pre-flight runs + /// unchanged against a host that has a GPU. + /// + /// **The runtime.** Black-box, like [`Self::register_mock_service`]: the + /// manifest is plain JSON matching the engine's on-disk schema, not a typed + /// import. `resolve_runtime` accepts it because it checks only that the + /// manifest parses and that `lemond` is a file; no checksum, version or + /// signature is on this path, so nothing here weakens a check the product + /// performs. `env_id` is the one field that cannot be a sentinel: `serve` + /// skips its own install only when `Detect` reports the id + /// `managed_engine_runtime_id` builds for the pinned version, and a + /// placeholder makes it quietly run the real install instead. So its + /// version is read from the workspace pin file, `runtime-deps.toml`, the + /// same file the product builds that id from, rather than copied. + /// + /// It goes under the scenario's own `data/engines/lemonade`, because this + /// scenario does not call [`Self::use_shared_runtimes`]: with an empty + /// runtimes registry the CLI resolves no env root, so `lemonade_root` is the + /// isolated engine dir. + /// + /// **The GPU.** One KFD GPU node under a simulated host root + /// (`ROCM_CLI_TEST_HOST_ROOT`), which is what `rocm serve`'s pre-flight + /// counts. The root holds nothing else, so a GPU runner's real devices do + /// not take part either. The host root is honoured only by the + /// `e2e-test-hooks` build and re-roots Linux paths only, which is why the + /// scenario is `@requires-os:linux`. The serve must therefore run through + /// `run_rocm_with_scenario_env`. + pub fn plant_failing_lemonade_runtime(&mut self) { + let root = self + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let engine = root.join("data").join("engines").join("lemonade"); + let runtime_dir = engine.join("runtime"); + // The fixture binaries go under `runtime/` and the manifest under + // `manifests/`; `resolve_runtime` reads that manifest and requires its + // `lemond` to be a file. The engine's launch creates its own state and + // log directories, so they are not planted here. + for dir in [&runtime_dir, &engine.join("manifests")] { + std::fs::create_dir_all(dir) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", dir.display())); + } + + // One fixture under both names; it takes its role from argv[0]. + let lemond = runtime_dir.join(e2e_cucumber::fixture_binary::fixture_target_name("lemond")); + let lemonade = runtime_dir.join(e2e_cucumber::fixture_binary::fixture_target_name( + "lemonade", + )); + install_fixture_binary_as("fake-lemonade", &[&lemond, &lemonade]); + + let version = e2e_cucumber::runtime_pins::pinned_version("lemonade"); + let manifest = serde_json::json!({ + // Mirrors `rocm_deps::lemonade_env_id`, which the engine and `rocm` + // both use; the suite cannot import it. A drift sends `serve` to + // the real install, which serve-18's refusal count catches. + "env_id": format!("lemonade-embeddable-{version}"), + "version": version, + "runtime_dir": runtime_dir, + "lemond": lemond, + "lemonade": lemonade, + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + }); + let path = engine.join("manifests").join("runtime.json"); + std::fs::write( + &path, + serde_json::to_vec_pretty(&manifest).expect("manifest json"), + ) + .unwrap_or_else(|e| panic!("failed to write {}: {e}", path.display())); + + let host_root = root.join("host-root"); + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + "cpu_cores_count 0\nsimd_count 120\ngfx_target_version 110001\n", + ) + .expect("failed to write planted KFD node properties"); + self.command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); + + // The engine prepares a runtime directory for `lemond` under + // `XDG_RUNTIME_DIR`, falling back to the real `$HOME`. Give it one inside + // the scenario, so the fake server writes nothing outside the isolated + // tree and a runner whose `$HOME` is not writable still gets this far. + let runtime = root.join("runtime"); + std::fs::create_dir_all(&runtime).expect("failed to create isolated runtime dir"); + self.command_env + .push(("XDG_RUNTIME_DIR", runtime.into_os_string())); + } + /// Register the running mock server with the CLI by writing a managed-service /// record into the isolated services directory (`/services/`), exactly /// as `rocm serve --managed` would. This lets `rocm services list` and the @@ -933,18 +1037,25 @@ pub fn rocmd_binary() -> PathBuf { /// and `rocm-core` is a dev-dependency this crate's library target cannot /// reach, only this test binary. pub fn install_fixture_binary(name: &str, dest: &Path) { - if let Some(parent) = dest.parent() { - std::fs::create_dir_all(parent) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", parent.display())); - } + install_fixture_binary_as(name, &[dest]); +} + +/// [`install_fixture_binary`] under several names, reading the fixture once. +pub fn install_fixture_binary_as(name: &str, dests: &[&Path]) { let contents = std::fs::read(e2e_cucumber::fixture_binary::fixture_binary(name)) .unwrap_or_else(|e| panic!("failed to read the {name} stand-in: {e}")); - rocm_core::fake_executable::write_fake_executable(dest, contents).unwrap_or_else(|e| { - panic!( - "failed to install the {name} stand-in at {}: {e}", - dest.display() - ) - }); + for dest in dests { + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", parent.display())); + } + rocm_core::fake_executable::write_fake_executable(dest, &contents).unwrap_or_else(|e| { + panic!( + "failed to install the {name} stand-in at {}: {e}", + dest.display() + ) + }); + } } /// Spawn the real `rocm` binary with the scenario's isolated env. diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 990bb6b188..098c95016f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use std::fmt::Write as _; use std::path::PathBuf; use std::sync::LazyLock; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -18,8 +19,8 @@ use e2e_cucumber::device_memory::{ }; use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ - ServeAttempt, archive_service_log, engine_from_serve_args, reported_engine, - serve_attempt_report, service_log_tail, + EngineDeathReport, ServeAttempt, archive_service_log, engine_death_report, + engine_from_serve_args, reported_engine, serve_attempt_report, service_log_tail, }; use e2e_cucumber::serve_plan::{ServeRun, resolved_model}; @@ -1402,15 +1403,30 @@ async fn user_serves_vllm_capable_default(world: &mut E2eWorld) { #[given("Lemonade preparation cannot complete")] async fn lemonade_preparation_cannot_complete(world: &mut E2eWorld) { - world.command_env.push(( - "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", - "repeated".into(), - )); -} - + // A planted runtime whose `lemonade backends install` always exits + // non-zero, on a simulated host with one GPU: the CLI fails at the boundary + // a broken install fails at, rather than at a branch compiled into it. + world.plant_failing_lemonade_runtime(); +} + +/// `--wait` is load-bearing. The backend install fails inside the managed +/// engine process, after `rocm serve` has spawned it. Without `--wait`, serve +/// only notices an engine that dies within its 200 ms post-spawn check; one +/// that dies later is reported as `starting` with exit 0 once the readiness +/// probe gives up, so on a loaded host this scenario would race that window. +/// With `--wait` a later death fails the serve too, once the 45 s readiness +/// probe ends, so the slow path costs that long. +/// +/// Both ways the failure reaches the user are followed: an engine that died +/// inside the window fails the serve with its log tail, and one that died +/// later fails it with a pointer to `rocm logs --service `, which this step +/// then runs, as the user would. The retry and recovery lines are the engine's +/// own output either way, so the assertions read the serve output together +/// with that log. A failed serve carrying neither message fails here, naming +/// both, rather than as a missing retry line later. #[when("the user serves a model with Lemonade")] async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { - let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + let (mut stdout, stderr, rc) = crate::run_rocm_with_scenario_env( world, &[ "serve", @@ -1418,8 +1434,30 @@ async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { "--engine", "lemonade", "--managed", + "--wait", ], ); + if rc != 0 && !stderr.contains("no usable AMD GPU detected") { + match engine_death_report(&stderr) { + EngineDeathReport::LogTailQuoted => {} + EngineDeathReport::LogsPointer(service_id) => { + let (logs, logs_stderr, logs_rc) = + crate::run_rocm(world, &["logs", "--service", &service_id]); + assert_eq!( + logs_rc, 0, + "`rocm logs --service {service_id}`, which the failed serve pointed at, \ + failed:\n{logs}\n{logs_stderr}" + ); + let _ = write!(stdout, "\n--- rocm logs --service {service_id} ---\n{logs}"); + } + EngineDeathReport::Unrecognised => panic!( + "serve failed without either engine-death message: neither the \ + post-spawn check's \"managed engine exited immediately\" log tail nor \ + the readiness wait's \"see `rocm logs --service `\" pointer. If the \ + CLI reworded one, update `engine_death_report`:\n{stdout}\n{stderr}" + ), + } + } world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); world.cli_rc = Some(rc); @@ -2161,8 +2199,8 @@ async fn assert_no_runtime_manifest_written(world: &mut E2eWorld) { /// Serve a Lemonade model under the GPU-required default while the unrelated /// live service planted above exists. No fault-injection env var: the scripted -/// Lemonade backend failure would waive the very GPU pre-flight this scenario -/// asserts. +/// Lemonade free-space override would waive the very GPU pre-flight this +/// scenario asserts. #[when("the user serves a different model with Lemonade under the GPU-required default")] async fn user_serves_other_model_with_unrelated_service_live(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm( @@ -2486,15 +2524,16 @@ async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { Some(0), "serve unexpectedly succeeded:\n{output}" ); - // The scripted failure seam also waives serve's no-GPU pre-flight, so this - // refusal can only appear when the seam is compiled out. Name that cause: - // otherwise a lane that pre-builds `rocm` without the feature reports a - // baffling "no retry announcement" instead of its real misconfiguration. + // The pre-flight counts the GPU planted under the simulated host root, which + // only an `e2e-test-hooks` build honours. Name that cause: otherwise a lane + // that pre-builds `rocm` without the feature reports a baffling "no retry + // announcement" instead of its real misconfiguration. assert!( !output.contains("no usable AMD GPU detected"), - "serve stopped at the no-GPU pre-flight, so the binary under test was \ - built without the `rocm/e2e-test-hooks` feature and never reached \ - Lemonade preparation. E2E lanes that pre-build `rocm` and export \ + "serve stopped at the no-GPU pre-flight, so the binary under test did not \ + see the GPU planted under ROCM_CLI_TEST_HOST_ROOT and never reached \ + Lemonade preparation. It was probably built without the \ + `rocm/e2e-test-hooks` feature: E2E lanes that pre-build `rocm` and export \ ROCM_CLI_BINARY must pass `--features rocm/e2e-test-hooks`, matching \ what `cargo xtask e2e` builds for itself:\n{output}" ); @@ -2503,6 +2542,16 @@ async fn assert_lemonade_preparation_retry_is_bounded(world: &mut E2eWorld) { 1, "expected exactly one retry announcement:\n{output}" ); + // Both attempts must have been refused by the planted runtime. Without its + // line, `serve` did not accept the planted runtime and attempted some other + // install -- for a mismatched `env_id`, the real multi-gigabyte one. + assert_eq!( + output + .matches(e2e_cucumber::fixture_binary::FAKE_LEMONADE_INSTALL_REFUSAL) + .count(), + 2, + "expected the planted runtime to refuse both backend installs:\n{output}" + ); } #[then("the user is told how to reinstall Lemonade and retry serving")] diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 9543c92115..f93e8281fa 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -211,14 +211,21 @@ fn package_args<'a>(packages: Option>) -> Vec bool { diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index d71237d321..915d19f1c2 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -269,8 +269,8 @@ mod tests { /// Every lane that pre-builds `rocm` and hands it to the suite via /// `ROCM_CLI_BINARY` must enable `rocm/e2e-test-hooks`. /// - /// The suite's deterministic failure seams (e.g. the scripted Lemonade - /// backend-install failure) are `#[cfg(feature = "e2e-test-hooks")]`. A lane + /// The suite's test seams (e.g. the simulated host root, + /// `ROCM_CLI_TEST_HOST_ROOT`) are `#[cfg(feature = "e2e-test-hooks")]`. A lane /// that omits the feature ships a binary in which those seams do not exist, /// so the scenarios relying on them cannot reach their premise and fail as /// regressions — but only on whichever lane happens to select them, which is @@ -300,8 +300,8 @@ mod tests { ), "{workflow} prebuilt E2E lane must build with \ `--features rocm/e2e-test-hooks`, the hook floor `cargo xtask e2e` \ - also builds for itself; without it the suite's scripted failure \ - seams are compiled out:\n{block}" + also builds for itself; without it the suite's test seams, such as \ + the simulated host root, are compiled out:\n{block}" ); } }