From 7f1ddf40aef7fda507162dff2fb9e038d440b62c Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 16:29:49 +0200 Subject: [PATCH 1/6] fix(lemonade): stop retrying a failed backend alignment on every install (ROCMAI-437) When neither alignment tier could match Lemonade's ROCm backend to the active ROCm version, the revert put back a pin that still differed from that version and nothing recorded the failure. Every later `rocm engines install lemonade`, and every `rocm serve` that installs Lemonade, then ran both tiers again, re-downloaded the tier-2 archive and force-reinstalled the packaged backend; a nightly GPU lane showed this five times in a single job. A failed alignment is now recorded in the Lemonade runtime directory. For 24 hours an install for the same active ROCm version, pin and Lemonade version skips the alignment, reuses the backend already installed, and prints when it failed and how to retry: `rocm engines install lemonade --reinstall`, which deletes the runtime tree and the record with it. A successful alignment removes the record; an unreadable or unwritable record warns and falls back to trying again. Scenario runtime-33 runs the two installs on the GPU nightly lanes, targeting an unpublished ROCm version through the e2e-test-hooks override ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION. Signed-off-by: Roman Inflianskas --- README.md | 7 +- docs/architecture.md | 5 +- docs/engine-plugins.md | 21 + docs/manual-testing.md | 18 + docs/testing.md | 27 + engines/lemonade/src/alignment_record.rs | 290 ++++++++++ engines/lemonade/src/backend_alignment.rs | 518 +++++++++++++++++- engines/lemonade/src/install.rs | 70 +++ engines/lemonade/src/lib.rs | 1 + .../features/runtime_setup.feature | 27 + .../tests/e2e/lemonade_install_steps.rs | 112 ++++ 11 files changed, 1089 insertions(+), 7 deletions(-) create mode 100644 engines/lemonade/src/alignment_record.rs diff --git a/README.md b/README.md index 49924aab2d..dbd3fe59d0 100644 --- a/README.md +++ b/README.md @@ -975,9 +975,12 @@ with the ROCm SDK in use, and rewrites the two pins it owns (`therock.version` and `llamacpp.rocm-stable` in `resources/backend_versions.json`) to do so. If installing an already-pinned backend fails, it restores those two pins from the cached Lemonade archive (it does not download), prints a warning, -and installs once more; see +and installs once more. When the backend cannot be aligned to the active ROCm +version, the install keeps the backend Lemonade pins, and for the next 24 hours +later installs reuse it without trying the alignment again; +`rocm engines install lemonade --reinstall` tries it again sooner. See [Lemonade backend alignment](https://github.com/ROCm/rocm-cli/blob/main/docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install) for the details and for -`ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, which turns both off. +`ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, which turns all of this off. Before Lemonade writes a new install, whether from `rocm engines install lemonade` or the first `rocm serve` that needs it, it prints the folder it is diff --git a/docs/architecture.md b/docs/architecture.md index 9275d8fbda..654348ca28 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -654,9 +654,10 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | --- | --- | --- | | [`lib.rs`](../engines/lemonade/src/lib.rs) | Crate root: CLI and envelope dispatch, the service-lifecycle orchestration that calls across the modules, and the crate-root re-exports `apps/rocm` uses. | flat | | [`main.rs`](../engines/lemonade/src/main.rs) | The `rocm-engine-lemonade` binary's entry point; it only calls `rocm_engine_lemonade::run_cli()`. | flat | +| [`alignment_record.rs`](../engines/lemonade/src/alignment_record.rs) | The record, inside the runtime directory, that aligning the llama.cpp backend to a ROCm version failed: reading, writing and clearing it, whether it still skips the same alignment (24 hours), and the line printed when it does. | domain | | [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) | The cached embeddable-archive layer: SHA-256 verification, verified publication into the cache, and the per-archive lock `ensure_cached_archive` returns as an `ArchiveCacheGuard` to hold while using the bytes. | domain | | [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) | The hardened in-process tar.gz and zip extractor for the embeddable archive, and the containment-checked `copy_tree` into the runtime directory. | domain | -| [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) | Aligning the llama.cpp backend with the ROCm SDK (Tier 1, Tier 2, revert), including the `backend_versions.json` pin and setting ROCm backend directories aside around a forced reinstall. | domain | +| [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) | Aligning the llama.cpp backend with the ROCm SDK (Tier 1, Tier 2, revert), including the `backend_versions.json` pin, setting ROCm backend directories aside around a forced reinstall, and skipping an alignment [`alignment_record.rs`](../engines/lemonade/src/alignment_record.rs) recorded as failed. | domain | | [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) | Device policy and GPU selection: the GPU-required check, `--device`/`--gpu` parsing, the GPU ordinals to pin before a launch, the device availability `detect` reports, and the notice that names a non-ROCm GPU backend (Vulkan) as the one serving. | domain | | [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) | Hugging Face checkpoint and GGUF resolution, including a registered model name resolved through the installed Lemonade catalog, which route a name serves by, and the direct serve path that runs a packaged `llama-server` on a GGUF (`serve_direct_llama_server`). | domain | | [`footprint.rs`](../engines/lemonade/src/footprint.rs) | What a Lemonade install costs on disk: the pre-install notice and free-space check, and the read-only view of Lemonade's install trees and record. | domain | @@ -854,7 +855,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`tests/e2e/engines_steps.rs`](../tests/e2e-cucumber/tests/e2e/engines_steps.rs) | Steps for [`engine_shell.feature`](../tests/e2e-cucumber/features/engine_shell.feature): `rocm engines shell` under a pseudo-terminal, since the defect shows only in what the terminal renders. | flat | | [`tests/e2e/examine_steps.rs`](../tests/e2e-cucumber/tests/e2e/examine_steps.rs) | Steps for [`examine.feature`](../tests/e2e-cucumber/features/examine.feature): `rocm examine`, version and help output, engine listing and the driver plan preview. | flat | | [`tests/e2e/host_guard.rs`](../tests/e2e-cucumber/tests/e2e/host_guard.rs) | The host check the Windows driver-install steps run before any CLI call, kept free of the cucumber world so a separate test can include it. | flat | -| [`tests/e2e/lemonade_install_steps.rs`](../tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs) | Steps for [`lemonade_installs.feature`](../tests/e2e-cucumber/features/lemonade_installs.feature): what the user is told about the Lemonade copies on disk, from planted trees and install records. | flat | +| [`tests/e2e/lemonade_install_steps.rs`](../tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs) | Steps for [`lemonade_installs.feature`](../tests/e2e-cucumber/features/lemonade_installs.feature): what the user is told about the Lemonade copies on disk, from planted trees and install records; also the steps of runtime-33 in [`runtime_setup.feature`](../tests/e2e-cucumber/features/runtime_setup.feature), where a Lemonade backend that could not be aligned is not tried again by the next install. | flat | | [`tests/e2e/lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs) | Steps for [`install_lifecycle.feature`](../tests/e2e-cucumber/features/install_lifecycle.feature): `xtask package`, the root installer and the installed binaries, from install through uninstall. | flat | | [`tests/e2e/logs_steps.rs`](../tests/e2e-cucumber/tests/e2e/logs_steps.rs) | Steps for [`logs.feature`](../tests/e2e-cucumber/features/logs.feature): `rocm logs` searches over command-log files planted inside the tail window it reads. | flat | | [`tests/e2e/model_catalog_steps.rs`](../tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs) | Steps for [`model_catalog.feature`](../tests/e2e-cucumber/features/model_catalog.feature): the verbose catalog's AIM entry, its Docker-unavailable flag and its staleness warning. | flat | diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 3cd9610703..48f3cb37d9 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -66,6 +66,27 @@ was pinned before the attempt — the packaged pin on a fresh install or after `--reinstall`, but whatever alignment last wrote otherwise — rather than being kept. +A failed alignment is remembered so that it is not repeated on every install. +The revert puts back a pin that still differs from the active ROCm version, so +without that every later `rocm engines install lemonade`, and every `rocm +serve` that installs Lemonade, would download both builds and force-reinstall +the pinned backend again. When the revert's install succeeds, the attempt (the +active ROCm version, the pins it started from, the llama.cpp builds tried and +the Lemonade version) is recorded in `.rocm-cli-alignment-failed.json` in the +Lemonade runtime tree. For the next 24 hours an install for the same active +ROCm version, pin and Lemonade version does not attempt the alignment: it +installs the backend already pinned without forcing a reinstall, so a backend +already installed is reused, and prints a line saying the alignment could not +be done, when it failed, and when it is tried again. `rocm engines install +lemonade --reinstall` tries it again at once, because it deletes and +re-extracts the runtime tree, record included. A different active ROCm version, +a new Lemonade version (which also re-extracts the tree), or a record older than +24 hours runs the alignment as before, and an alignment that succeeds removes +the record. A record that cannot be read is ignored with a warning, and one +that cannot be written leaves the next install to try again, also with a +warning. Setting `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` neither reads nor +changes the record. + A pin an earlier alignment kept can go stale: the llama.cpp or TheRock build it names may later be withdrawn upstream. When the pin already equals the active ROCm version the install does not attempt another alignment, so if installing diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 3f5124e992..396c02cafd 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -453,6 +453,24 @@ ROCm backend directory under `bin/llamacpp/`, and run - `therock.version` and `llamacpp.rocm-stable` in `backend_versions.json` hold the packaged values again, and the unrelated key is still there. +To check that a failed backend alignment is not repeated (Windows or native +Linux with a GPU Lemonade runs its ROCm backend on, and an active managed ROCm +SDK): build with `cargo build -p rocm --features rocm/e2e-test-hooks`, set +`ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION=7.99.0` (a ROCm version Lemonade +publishes no backend for), and run `rocm engines install lemonade --reinstall`, +then `rocm engines install lemonade`. Expected result: + +- The first install warns `could not align Lemonade's ROCm backend to 7.99.0; + reverting to the default pinned version`, exits 0, and leaves + `.rocm-cli-alignment-failed.json` in the Lemonade runtime tree. +- The second install prints no alignment warning and does not download or + reinstall the backend (`Using installed Lemonade llamacpp:rocm backend.`). It + prints `Lemonade's ROCm backend could not be aligned to ROCm 7.99.0 less than + an hour ago; ...`, naming `rocm engines install lemonade --reinstall` and + when the alignment is tried again, and exits 0. +- Running the first command again attempts the alignment again (the warning + is back), because `--reinstall` removed the record with the runtime tree. + To check what a Lemonade install says about its size (no download needed for the refusal): with a debug build, set `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES=1048576` (1 MiB free), then in a fresh data diff --git a/docs/testing.md b/docs/testing.md index a21272ebc1..505e0d99a0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2390,6 +2390,33 @@ lemonade` against the shared tree) reaches the restore only on a runner whose shared tree still holds a stale pin; on a healthy runner the install succeeds and the restore never runs, so a green lane does not show that it works. +Lemonade failed-alignment record focused tests (ROCMAI-437; a failed +alignment is not repeated by the next install for 24 hours; see +[Lemonade backend alignment on engine install](engine-plugins.md#lemonade-backend-alignment-on-engine-install)): + +```bash +cargo test -p rocm-engine-lemonade record +cargo test -p rocm-engine-lemonade a_failed_alignment_is_not_repeated +cargo test -p rocm-engine-lemonade a_skipped_alignment_whose_install_fails +``` + +The tests are in [alignment_record.rs](../engines/lemonade/src/alignment_record.rs) +and [backend_alignment.rs](../engines/lemonade/src/backend_alignment.rs), one per +state of the record: none, one that skips the alignment +(the regression test: the second install calls no alignment tier and installs +without forcing, and its notice is asserted with the pin file), one for another +ROCm version, one 24 hours old, an unreadable one, one that cannot be written, +one removed by a later successful alignment, one removed by `--reinstall` +(`reinstall_removes_the_record_of_a_failed_alignment` in +[install.rs](../engines/lemonade/src/install.rs)), one the +opt-out leaves alone, and a skipped alignment whose install fails (the error is +reported and no notice is printed). The scenario +`runtime-lemonade-backend-alignment-failure-remembered` (runtime-33 in +[runtime_setup.feature](../tests/e2e-cucumber/features/runtime_setup.feature)) runs the two installs for real on the +`@nightly @requires-gpu` Lemonade lanes (not under WSL2); it targets ROCm 7.99.0 +through `ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION`, compiled only into the +`rocm/e2e-test-hooks` build. + Engine recipe adapter contract focused tests: ```bash diff --git a/engines/lemonade/src/alignment_record.rs b/engines/lemonade/src/alignment_record.rs new file mode 100644 index 0000000000..90280bc85c --- /dev/null +++ b/engines/lemonade/src/alignment_record.rs @@ -0,0 +1,290 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The record that an attempt to align Lemonade's ROCm backend failed. +//! +//! An alignment that fails costs two backend downloads and a forced reinstall of +//! the backend Lemonade ships. Its revert leaves the pin different from the active +//! ROCm SDK, so without a record every later install would repeat all of it. The +//! record lives inside the Lemonade runtime tree it describes, so a +//! `--reinstall` (or a new Lemonade version), which deletes and re-extracts that +//! tree, removes it as well. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// File name of the record, inside the Lemonade runtime directory. +pub(crate) const ALIGNMENT_FAILURE_RECORD: &str = ".rocm-cli-alignment-failed.json"; + +/// How long a recorded failure stops the same alignment from being tried again. +pub(crate) const ALIGNMENT_RETRY_AFTER: Duration = Duration::from_hours(24); + +/// The command that tries the alignment again before the record expires. It +/// deletes the runtime tree, record included, so the next alignment runs. +pub(crate) const RETRY_ALIGNMENT_COMMAND: &str = "rocm engines install lemonade --reinstall"; + +/// What one failed alignment attempt tried, and when it failed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct AlignmentFailure { + /// The active ROCm SDK version the backend could not be aligned to. + pub(crate) target_rocm_version: String, + /// The ROCm version `backend_versions.json` pinned before the attempt, which + /// the revert put back. + pub(crate) pinned_rocm_version: String, + /// The llama.cpp build tag pinned before the attempt (tier 1 tried it). + pub(crate) pinned_llamacpp_tag: Option, + /// The newest llama.cpp build tag tier 2 looked up, when the lookup worked. + pub(crate) latest_llamacpp_tag: Option, + /// The Lemonade version whose runtime tree the attempt ran in. + pub(crate) lemonade_version: String, + /// When the attempt failed, in seconds since the Unix epoch. + pub(crate) failed_at_unix_secs: u64, +} + +/// What the runtime directory holds in place of a record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum StoredAlignmentFailure { + Absent, + /// A record file exists but cannot be read or parsed; the text says why. + Unreadable(String), + Recorded(AlignmentFailure), +} + +pub(crate) fn alignment_failure_path(runtime_dir: &Path) -> PathBuf { + runtime_dir.join(ALIGNMENT_FAILURE_RECORD) +} + +pub(crate) fn read_alignment_failure(runtime_dir: &Path) -> StoredAlignmentFailure { + let path = alignment_failure_path(runtime_dir); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == ErrorKind::NotFound => { + return StoredAlignmentFailure::Absent; + } + Err(error) => { + return StoredAlignmentFailure::Unreadable(format!( + "could not read {}: {error}", + path.display() + )); + } + }; + match serde_json::from_slice(&bytes) { + Ok(record) => StoredAlignmentFailure::Recorded(record), + Err(error) => StoredAlignmentFailure::Unreadable(format!( + "could not parse {}: {error}", + path.display() + )), + } +} + +pub(crate) fn write_alignment_failure(runtime_dir: &Path, record: &AlignmentFailure) -> Result<()> { + let path = alignment_failure_path(runtime_dir); + rocm_core::atomic_write::write_file_atomically(&path, &serde_json::to_vec_pretty(record)?) + .with_context(|| format!("failed to write {}", path.display())) +} + +/// Remove the record. A missing record is not an error. +pub(crate) fn clear_alignment_failure(runtime_dir: &Path) -> Result<()> { + let path = alignment_failure_path(runtime_dir); + match fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error).with_context(|| format!("failed to remove {}", path.display())), + } +} + +/// Seconds since the Unix epoch; zero for a clock set before it. +pub(crate) fn unix_secs(time: SystemTime) -> u64 { + time.duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_secs()) +} + +impl AlignmentFailure { + /// Whether the alignment should be skipped now: the same attempt (target, + /// pin and Lemonade version) failed less than [`ALIGNMENT_RETRY_AFTER`] ago. + /// + /// A failure time later than `now` (the clock moved back) does not skip: + /// trying again costs an install, and a record whose age cannot be known + /// must not stop the alignment indefinitely. + pub(crate) fn skips_alignment( + &self, + target_rocm_version: &str, + pinned_rocm_version: &str, + lemonade_version: &str, + now: SystemTime, + ) -> bool { + self.target_rocm_version == target_rocm_version + && self.pinned_rocm_version == pinned_rocm_version + && self.lemonade_version == lemonade_version + && self.age(now).is_some_and(|age| age < ALIGNMENT_RETRY_AFTER) + } + + fn age(&self, now: SystemTime) -> Option { + let failed_at = UNIX_EPOCH.checked_add(Duration::from_secs(self.failed_at_unix_secs))?; + now.duration_since(failed_at).ok() + } + + /// The line printed when this record skipped the alignment and the pinned + /// backend was installed instead. Built from the record and the same `now` + /// [`Self::skips_alignment`] was asked with, so the times it names are the + /// ones the decision used. + pub(crate) fn skip_notice(&self, now: SystemTime) -> String { + let age = self.age(now).unwrap_or_default(); + let remaining = ALIGNMENT_RETRY_AFTER.saturating_sub(age); + format!( + "Lemonade's ROCm backend could not be aligned to ROCm {target} {ago}; using the \ + backend pinned for ROCm {pinned} instead. Run `{RETRY_ALIGNMENT_COMMAND}` to try \ + again now, or the alignment is tried again {when}.", + target = self.target_rocm_version, + ago = describe_ago(age), + pinned = self.pinned_rocm_version, + when = describe_in(remaining), + ) + } +} + +const HOUR: u64 = 60 * 60; + +/// "less than an hour ago", "1 hour ago", "5 hours ago": whole hours, rounded down. +fn describe_ago(age: Duration) -> String { + match age.as_secs() / HOUR { + 0 => "less than an hour ago".to_owned(), + 1 => "1 hour ago".to_owned(), + hours => format!("{hours} hours ago"), + } +} + +/// "in less than an hour", "in 1 hour", "in 5 hours": whole hours, rounded up +/// so it never names a time before the record expires. +fn describe_in(remaining: Duration) -> String { + if remaining.as_secs() < HOUR { + return "in less than an hour".to_owned(); + } + match remaining.as_secs().div_ceil(HOUR) { + 1 => "in 1 hour".to_owned(), + hours => format!("in {hours} hours"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::scratch_dir; + + fn record(failed_at_unix_secs: u64) -> AlignmentFailure { + AlignmentFailure { + target_rocm_version: "7.99.0".to_owned(), + pinned_rocm_version: "7.13.0".to_owned(), + pinned_llamacpp_tag: Some("b9752".to_owned()), + latest_llamacpp_tag: Some("b10952".to_owned()), + lemonade_version: "11.5.1".to_owned(), + failed_at_unix_secs, + } + } + + fn at(secs: u64) -> SystemTime { + UNIX_EPOCH + Duration::from_secs(secs) + } + + const FAILED_AT: u64 = 1_800_000_000; + + #[test] + fn a_record_round_trips_and_clearing_it_twice_is_fine() { + let dir = scratch_dir("alignment-record-round-trip"); + assert_eq!(read_alignment_failure(&dir), StoredAlignmentFailure::Absent); + + write_alignment_failure(&dir, &record(FAILED_AT)).unwrap(); + assert_eq!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Recorded(record(FAILED_AT)) + ); + + clear_alignment_failure(&dir).unwrap(); + assert!(!alignment_failure_path(&dir).exists()); + clear_alignment_failure(&dir).unwrap(); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_corrupt_record_reads_as_unreadable_not_absent() { + let dir = scratch_dir("alignment-record-corrupt"); + fs::write(alignment_failure_path(&dir), b"{ not json").unwrap(); + + let StoredAlignmentFailure::Unreadable(reason) = read_alignment_failure(&dir) else { + panic!("a corrupt record must read as unreadable"); + }; + assert!(reason.contains(ALIGNMENT_FAILURE_RECORD), "{reason}"); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_record_skips_only_the_same_attempt_within_the_retry_window() { + let failure = record(FAILED_AT); + let skips = + |target, pinned, lemonade, now| failure.skips_alignment(target, pinned, lemonade, now); + let window = ALIGNMENT_RETRY_AFTER.as_secs(); + + assert!(skips("7.99.0", "7.13.0", "11.5.1", at(FAILED_AT))); + assert!(skips( + "7.99.0", + "7.13.0", + "11.5.1", + at(FAILED_AT + window - 1) + )); + assert!( + !skips("7.99.0", "7.13.0", "11.5.1", at(FAILED_AT + window)), + "a record exactly 24 hours old has expired" + ); + assert!( + !skips("7.98.0", "7.13.0", "11.5.1", at(FAILED_AT)), + "other target" + ); + assert!( + !skips("7.99.0", "7.12.0", "11.5.1", at(FAILED_AT)), + "other pin" + ); + assert!( + !skips("7.99.0", "7.13.0", "11.6.0", at(FAILED_AT)), + "other Lemonade" + ); + assert!( + !skips("7.99.0", "7.13.0", "11.5.1", at(FAILED_AT - 1)), + "a failure time in the future does not skip" + ); + } + + #[test] + fn the_skip_notice_names_the_target_the_pin_the_command_and_the_times() { + let failure = record(FAILED_AT); + + let fresh = failure.skip_notice(at(FAILED_AT + 60)); + assert!( + fresh.contains("aligned to ROCm 7.99.0 less than an hour ago"), + "{fresh}" + ); + assert!(fresh.contains("pinned for ROCm 7.13.0"), "{fresh}"); + assert!( + fresh.contains("`rocm engines install lemonade --reinstall`"), + "{fresh}" + ); + assert!(fresh.contains("tried again in 24 hours"), "{fresh}"); + + let later = failure.skip_notice(at(FAILED_AT + 5 * HOUR + 1)); + assert!(later.contains("5 hours ago"), "{later}"); + assert!(later.contains("tried again in 19 hours"), "{later}"); + + let one = failure.skip_notice(at(FAILED_AT + HOUR)); + assert!(one.contains("1 hour ago"), "{one}"); + assert!(one.contains("in 23 hours"), "{one}"); + + let last = failure.skip_notice(at(FAILED_AT + 23 * HOUR + 1)); + assert!(last.contains("in less than an hour"), "{last}"); + let penultimate = failure.skip_notice(at(FAILED_AT + 23 * HOUR)); + assert!(penultimate.contains("in 1 hour"), "{penultimate}"); + } +} diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index dc07423f1d..bb9d0592f7 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -13,9 +13,13 @@ use std::fs; use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::{Command as ProcessCommand, Stdio}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use crate::ROCM_BACKEND_NAME; +use crate::alignment_record::{ + AlignmentFailure, StoredAlignmentFailure, clear_alignment_failure, read_alignment_failure, + unix_secs, write_alignment_failure, +}; use crate::install::{LemonadeInstallManifest, PinFileBackup}; use crate::llamacpp_backend::{ ROCM_LLAMACPP_BACKEND_DIRS, best_llamacpp_backend_for_host, @@ -68,6 +72,12 @@ fn lemonade_backend_alignment_disabled() -> bool { /// Lemonade ships and installs once more (a forced reinstall), so that branch can /// rewrite `backend_versions.json` and replace the selected backend. /// +/// An alignment that fails is recorded in the runtime tree +/// ([`crate::alignment_record`]), and for the next +/// [`ALIGNMENT_RETRY_AFTER`](crate::alignment_record::ALIGNMENT_RETRY_AFTER) the same +/// alignment is not tried again: the backend already pinned is installed without +/// forcing a reinstall, and a line says so and how to try again sooner. +/// /// Returns the aligned version string when one was applied and verified, so the caller /// can surface it in the install response. pub(crate) fn prepare_llamacpp_backend_for_active_rocm( @@ -78,6 +88,8 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( manifest, lemonade_backend_alignment_disabled(), !runtime_is_linux() && !runtime_is_windows(), + SystemTime::now, + |message| eprintln!("{message}"), || active_rocm_sdk_version_for_alignment(paths), read_backend_versions_therock_version, best_llamacpp_backend_for_host, @@ -95,6 +107,8 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( try_llamacpp_backend_alignment, install_best_llamacpp_backend, latest_llamacpp_rocm_stable_tag, + SystemTime::now, + &mut |message| eprintln!("Warning: {message}"), ) }, ) @@ -122,11 +136,17 @@ pub(crate) fn prepare_llamacpp_backend_for_active_rocm( /// false` here as well as there. `align`'s own `if disabled` branch exists only so /// its unit tests can pin the state machine's half of the contract (the pin and /// llama.cpp tag staying untouched) without duplicating this whole call chain. +/// +/// `now` is the clock the recorded-failure check reads, and `notice` receives the +/// lines that check prints, so a test can drive the 24-hour window and assert each +/// line together with the install it describes. #[allow(clippy::too_many_arguments)] fn prepare_llamacpp_backend_for_active_rocm_impl( manifest: &mut LemonadeInstallManifest, disabled: bool, unsupported_platform: bool, + now: impl Fn() -> SystemTime, + mut notice: impl FnMut(String), mut active_rocm_sdk_version: impl FnMut() -> Result>, mut pinned_version_lookup: impl FnMut(&Path) -> Option, mut selected_backend_lookup: impl FnMut(&LemonadeInstallManifest) -> Result>, @@ -197,6 +217,31 @@ fn prepare_llamacpp_backend_for_active_rocm_impl( ); } + // An earlier install already tried this same alignment and it failed. Its + // revert put the pin back to `pinned_version`, which still differs from the + // target, so without this check every install would download both tiers' + // builds and force-reinstall the pinned backend again. Checked before the + // backend probe below, which starts `lemond`: an alignment is only recorded as + // failed after that probe chose the ROCm backend. + match read_alignment_failure(&manifest.runtime_dir) { + StoredAlignmentFailure::Recorded(failure) => { + let now = now(); + if failure.skips_alignment(&target_version, &pinned_version, &manifest.version, now) { + // Not forced: the backend the revert installed is the one wanted. + // The line is printed only once that install has succeeded, since + // it says the pinned backend is in use. + fallback_install(manifest, false)?; + notice(failure.skip_notice(now)); + return Ok(None); + } + } + StoredAlignmentFailure::Unreadable(reason) => notice(format!( + "Warning: ignoring the record of an earlier failed Lemonade backend alignment \ + ({reason}); trying the alignment again." + )), + StoredAlignmentFailure::Absent => {} + } + // Verification below requires the backend Lemonade selects to be `rocm` // specifically ([`try_llamacpp_backend_alignment`]'s `has_usable_binary` gate) -- // it falls back to `vulkan` on hosts where Lemonade's ROCm build is unsupported @@ -355,6 +400,11 @@ fn install_pinned_backend_or_restore_packaged_pins( /// chain above it. The user-facing announcement lives in /// [`prepare_llamacpp_backend_for_active_rocm_impl`] as well, so it fires on every /// skip path the opt-out affects, not only the one this function's own gate reaches. +/// +/// A revert whose fallback install succeeds records the failure +/// ([`crate::alignment_record`]), stamped with `now`, so the next install does not +/// repeat it; a verified alignment removes any such record. `warn` receives the +/// warning printed when either fails, so a test asserts it with the file state. #[allow(clippy::too_many_arguments)] fn align_llamacpp_backend_to_version( manifest: &mut LemonadeInstallManifest, @@ -365,6 +415,8 @@ fn align_llamacpp_backend_to_version( mut align: impl FnMut(&mut LemonadeInstallManifest, &str, bool, &str) -> bool, mut fallback_install: impl FnMut(&mut LemonadeInstallManifest, bool) -> Result<()>, mut latest_tag: impl FnMut() -> Result, + now: impl Fn() -> SystemTime, + warn: &mut impl FnMut(String), ) -> Result> { if disabled { fallback_install(manifest, false)?; @@ -385,6 +437,7 @@ fn align_llamacpp_backend_to_version( // check above), so a backend already on disk was installed against the OLD // pin and must not be trusted just because it happens to still resolve. if align(manifest, target_version, true, "its pinned llama.cpp build") { + forget_alignment_failure(manifest, warn); return Ok(Some(target_version.to_owned())); } @@ -396,9 +449,10 @@ fn align_llamacpp_backend_to_version( // paired with the active ROCm version, forcing a real reinstall attempt // regardless of whatever Tier 1 left on disk. let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag_applied = match latest_tag() { + let latest_tag = latest_tag(); + let latest_tag_applied = match &latest_tag { Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { - match write_backend_versions_llamacpp_tag(backend_versions_path, &latest_tag) { + match write_backend_versions_llamacpp_tag(backend_versions_path, latest_tag) { Ok(()) => true, Err(error) => { eprintln!( @@ -417,6 +471,7 @@ fn align_llamacpp_backend_to_version( } }; if latest_tag_applied && align(manifest, target_version, true, "the latest llama.cpp build") { + forget_alignment_failure(manifest, warn); return Ok(Some(target_version.to_owned())); } @@ -459,6 +514,22 @@ fn align_llamacpp_backend_to_version( match fallback_install(manifest, true) { Ok(()) => { resolve_rocm_llamacpp_backend_dirs_aside(manifest, &aside_dirs, true); + // Recorded only once the pinned backend is back in place: the record + // tells the next install to reuse it rather than reinstall it. + let failure = AlignmentFailure { + target_rocm_version: target_version.to_owned(), + pinned_rocm_version: pinned_version.to_owned(), + pinned_llamacpp_tag: pinned_tag, + latest_llamacpp_tag: latest_tag.ok(), + lemonade_version: manifest.version.clone(), + failed_at_unix_secs: unix_secs(now()), + }; + if let Err(error) = write_alignment_failure(&manifest.runtime_dir, &failure) { + warn(format!( + "could not record that aligning Lemonade's ROCm backend to {target_version} \ + failed, so the next install tries it again: {error:#}" + )); + } Ok(None) } Err(error) => { @@ -470,6 +541,19 @@ fn align_llamacpp_backend_to_version( } } +/// Remove the record of an earlier failed alignment once an alignment has been +/// verified, so it cannot skip a later one. Best-effort: a record left behind is +/// not read while the pin equals the version it was aligned to, which is what a +/// verified alignment leaves. +fn forget_alignment_failure(manifest: &LemonadeInstallManifest, warn: &mut impl FnMut(String)) { + if let Err(error) = clear_alignment_failure(&manifest.runtime_dir) { + warn(format!( + "could not remove the record of an earlier failed Lemonade backend alignment: \ + {error:#}" + )); + } +} + /// Attempt one llama.cpp backend install aligned to `target_version`'s ROCm pairing /// (already written into `resources/backend_versions.json`), then verify — never /// simply trust Lemonade's own reported success — that the resulting GPU backend @@ -628,6 +712,13 @@ fn remove_backend_versions_llamacpp_tag(path: &Path) -> Result<()> { /// config would discard the active runtime key and let version selection guess the /// most-recently-installed runtime, then mutate Lemonade based on that guess. fn active_rocm_sdk_version_for_alignment(paths: &AppPaths) -> Result> { + // e2e-only: a black-box scenario names a ROCm version no Lemonade backend + // build exists for, so the alignment fails the same way on every GPU lane + // without planting a fake runtime in the runtime tree other scenarios share. + #[cfg(feature = "e2e-test-hooks")] + if let Some(version) = std::env::var_os(ALIGNMENT_TARGET_TEST_ENV) { + return Ok(Some(version.to_string_lossy().into_owned())); + } let config = RocmCliConfig::load(paths)?; let Some(version) = active_managed_therock_version(paths, &config)? else { return Ok(None); @@ -635,6 +726,11 @@ fn active_rocm_sdk_version_for_alignment(paths: &AppPaths) -> Result bool { @@ -1320,6 +1416,8 @@ mod tests { Ok(()) }, || panic!("disabled means tier 2's tag lookup must not run either"), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), None); @@ -1350,6 +1448,8 @@ mod tests { &mut manifest, true, false, + || SystemTime::UNIX_EPOCH, + |_| {}, || panic!("disabled means the active SDK version must never be probed"), |_path| panic!("disabled means the packaged pin must never be read"), |_manifest| panic!("disabled means the vulkan-fallback probe must never run"), @@ -1382,6 +1482,8 @@ mod tests { &mut manifest, false, true, + || SystemTime::UNIX_EPOCH, + |_| {}, || panic!("unsupported platform means the active SDK version must never be probed"), |_path| panic!("unsupported platform means the packaged pin must never be read"), |_manifest| { @@ -1416,6 +1518,8 @@ mod tests { &mut manifest, false, false, + || SystemTime::UNIX_EPOCH, + |_| {}, || Ok(Some("10.0.0".to_owned())), |_path| Some("7.13.0".to_owned()), |_manifest| Err(anyhow!("lemond failed to start")), @@ -1452,6 +1556,8 @@ mod tests { &mut manifest, false, false, + || SystemTime::UNIX_EPOCH, + |_| {}, || Ok(Some("7.14.1".to_owned())), |_path| Some("7.14.1".to_owned()), |_manifest| panic!("an equal pin never reaches the backend probe"), @@ -1917,6 +2023,8 @@ mod tests { }, |_manifest, _force_reinstall| panic!("tier 1 succeeded; no fallback install"), || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); @@ -1954,6 +2062,8 @@ mod tests { }, |_manifest, _force_reinstall| panic!("tier 2 succeeded; no fallback install"), || Ok("b10952".to_owned()), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), Some("10.0.0".to_owned())); @@ -1993,6 +2103,8 @@ mod tests { Ok(()) }, || Ok("b10952".to_owned()), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), None); @@ -2039,6 +2151,8 @@ mod tests { Ok(()) }, || Ok("b10952".to_owned()), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), None); @@ -2077,6 +2191,8 @@ mod tests { }, // Already the latest tag: Tier 2's own reinstall must not be attempted. || Ok("b9752".to_owned()), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!(result.unwrap(), None); @@ -2122,6 +2238,8 @@ mod tests { Ok(()) }, || bail!("network unavailable"), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert_eq!( @@ -2162,6 +2280,8 @@ mod tests { bail!("network unavailable") }, || Ok("b10952".to_owned()), + || SystemTime::UNIX_EPOCH, + &mut |_| {}, ); assert!( @@ -2291,4 +2411,396 @@ mod tests { fs::remove_file(path_dir.join("hipblas.dll")).unwrap(); assert!(!windows_backend_resolves(&backend_dir, &search_dirs)); } + + // The record of a failed alignment (ROCMAI-437). Each test below is one row of + // the record's state matrix: no record, a record that applies, one for another + // target, an expired one, an unreadable one, a record that cannot be written, + // an alignment that later succeeds, the opt-out, and an install that fails + // after the record skipped the alignment. `--reinstall` removing the record is + // in `install.rs` (`reinstall_removes_the_record_of_a_failed_alignment`). + + const RECORD_TARGET: &str = "7.99.0"; + const RECORD_PIN: &str = "7.13.0"; + const FAILED_AT: u64 = 1_800_000_000; + const HOUR: u64 = 60 * 60; + + fn at(secs: u64) -> SystemTime { + SystemTime::UNIX_EPOCH + Duration::from_secs(secs) + } + + fn recorded_failure(target: &str, failed_at_unix_secs: u64) -> AlignmentFailure { + AlignmentFailure { + target_rocm_version: target.to_owned(), + pinned_rocm_version: RECORD_PIN.to_owned(), + pinned_llamacpp_tag: Some("b9752".to_owned()), + latest_llamacpp_tag: Some("b10952".to_owned()), + lemonade_version: rocm_deps::LEMONADE_VERSION.to_owned(), + failed_at_unix_secs, + } + } + + /// A runtime directory whose `backend_versions.json` pins [`RECORD_PIN`]. + fn runtime_dir_with_packaged_pin(tag: &str) -> (PathBuf, PathBuf) { + let dir = scratch_dir(tag); + let pin_file = dir.join(BACKEND_VERSIONS_RESOURCE); + fs::create_dir_all(pin_file.parent().unwrap()).unwrap(); + write_backend_versions_fixture(&pin_file, RECORD_PIN, "b9752"); + (dir, pin_file) + } + + struct PrepareRun { + result: Result>, + align_calls: usize, + installs: Vec, + notices: Vec, + } + + /// Runs `prepare` for an active SDK at [`RECORD_TARGET`] against the real pin + /// file and record in `dir`, with a stand-in `align` that only counts its calls + /// and a fallback install that returns `install_result`. + fn prepare_against_record( + dir: &Path, + disabled: bool, + now: SystemTime, + install_result: fn() -> Result<()>, + ) -> PrepareRun { + let mut manifest = test_manifest(dir.to_path_buf()); + let mut align_calls = 0; + let mut installs = Vec::new(); + let mut notices = Vec::new(); + let result = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + disabled, + false, + || now, + |message| notices.push(message), + || Ok(Some(RECORD_TARGET.to_owned())), + read_backend_versions_therock_version, + |_manifest| Ok(Some(ROCM_BACKEND_NAME.to_owned())), + |_manifest, force_reinstall| { + installs.push(force_reinstall); + install_result() + }, + |_manifest| panic!("a pin that differs from the target never restores"), + |_manifest, _path, _target, _pinned, _disabled| { + align_calls += 1; + Ok(None) + }, + ); + PrepareRun { + result, + align_calls, + installs, + notices, + } + } + + #[test] + fn a_failed_alignment_is_not_repeated_by_the_next_install_within_a_day() { + // The regression: a failed alignment reverts the pin to a version that still + // differs from the active SDK, so before the record every later install ran + // both tiers and the forced reinstall again. The first prepare runs the real + // tier/revert state machine; the second must not reach it. + let (dir, pin_file) = runtime_dir_with_packaged_pin("record-not-repeated"); + let mut tier_attempts = 0; + let mut first_installs = Vec::new(); + let mut manifest = test_manifest(dir.clone()); + let first = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + false, + false, + || at(FAILED_AT), + |message| panic!("nothing is recorded yet, so nothing is noticed: {message}"), + || Ok(Some(RECORD_TARGET.to_owned())), + read_backend_versions_therock_version, + |_manifest| Ok(Some(ROCM_BACKEND_NAME.to_owned())), + |_manifest, _force| panic!("the first install aligns rather than installing directly"), + |_manifest| panic!("a pin that differs from the target never restores"), + |manifest, path, target, pinned, disabled| { + align_llamacpp_backend_to_version( + manifest, + path, + target, + pinned, + disabled, + |_manifest, _target, _force, _label| { + tier_attempts += 1; + false + }, + |_manifest, force_reinstall| { + first_installs.push(force_reinstall); + Ok(()) + }, + || Ok("b10952".to_owned()), + || at(FAILED_AT), + &mut |message| panic!("the record is writable: {message}"), + ) + }, + ); + assert_eq!(first.unwrap(), None); + assert_eq!(tier_attempts, 2, "the first install tries both tiers"); + assert_eq!( + first_installs, + [true], + "and force-reinstalls the pinned backend" + ); + assert_eq!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Recorded(recorded_failure(RECORD_TARGET, FAILED_AT)), + "the failure is recorded with what was tried" + ); + + let now = at(FAILED_AT + HOUR); + let second = prepare_against_record(&dir, false, now, || Ok(())); + + assert_eq!(second.result.unwrap(), None); + assert_eq!( + second.align_calls, 0, + "the recorded alignment is not tried again" + ); + assert_eq!( + second.installs, + [false], + "the pinned backend is reused, not force-reinstalled" + ); + assert_eq!( + second.notices, + [recorded_failure(RECORD_TARGET, FAILED_AT).skip_notice(now)] + ); + let notice = &second.notices[0]; + assert!( + notice.contains("aligned to ROCm 7.99.0 1 hour ago"), + "{notice}" + ); + assert!( + notice.contains("rocm engines install lemonade --reinstall"), + "{notice}" + ); + assert!(notice.contains("tried again in 23 hours"), "{notice}"); + assert_eq!( + read_backend_versions_therock_version(&pin_file).as_deref(), + Some(RECORD_PIN), + "the notice says the pinned backend is in use, and the pin is untouched" + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn with_no_record_the_alignment_runs_as_before() { + let (dir, _) = runtime_dir_with_packaged_pin("record-absent"); + + let run = prepare_against_record(&dir, false, at(FAILED_AT), || Ok(())); + + assert_eq!(run.result.unwrap(), None); + assert_eq!(run.align_calls, 1); + assert!( + run.installs.is_empty(), + "align owns every install on this path" + ); + assert!(run.notices.is_empty(), "{:?}", run.notices); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_record_for_another_rocm_version_does_not_skip_the_alignment() { + let (dir, _) = runtime_dir_with_packaged_pin("record-other-target"); + let other = recorded_failure("7.98.0", FAILED_AT); + write_alignment_failure(&dir, &other).unwrap(); + + let run = prepare_against_record(&dir, false, at(FAILED_AT + HOUR), || Ok(())); + + assert_eq!(run.result.unwrap(), None); + assert_eq!( + run.align_calls, 1, + "the active SDK changed, so the alignment runs" + ); + assert!(run.installs.is_empty()); + assert!(run.notices.is_empty(), "{:?}", run.notices); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_record_a_day_old_no_longer_skips_the_alignment() { + let (dir, _) = runtime_dir_with_packaged_pin("record-expired"); + write_alignment_failure(&dir, &recorded_failure(RECORD_TARGET, FAILED_AT)).unwrap(); + + let run = prepare_against_record(&dir, false, at(FAILED_AT + 24 * HOUR), || Ok(())); + + assert_eq!(run.result.unwrap(), None); + assert_eq!( + run.align_calls, 1, + "after 24 hours the alignment is tried again" + ); + assert!(run.installs.is_empty()); + assert!(run.notices.is_empty(), "{:?}", run.notices); + + let just_inside = + prepare_against_record(&dir, false, at(FAILED_AT + 24 * HOUR - 1), || Ok(())); + assert_eq!( + just_inside.align_calls, 0, + "one second earlier it is still skipped" + ); + assert_eq!(just_inside.installs, [false]); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn an_unreadable_record_is_ignored_with_one_warning_and_the_alignment_runs() { + let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt"); + fs::write( + dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD), + b"{ torn", + ) + .unwrap(); + + let run = prepare_against_record(&dir, false, at(FAILED_AT), || Ok(())); + + assert_eq!(run.result.unwrap(), None); + assert_eq!( + run.align_calls, 1, + "an unreadable record does not skip the alignment" + ); + assert_eq!(run.notices.len(), 1, "{:?}", run.notices); + assert!( + run.notices[0].starts_with("Warning: ignoring the record") + && run.notices[0].contains("trying the alignment again"), + "{:?}", + run.notices + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_skipped_alignment_whose_install_fails_reports_the_error_and_claims_nothing() { + let (dir, _) = runtime_dir_with_packaged_pin("record-skip-install-fails"); + let failure = recorded_failure(RECORD_TARGET, FAILED_AT); + write_alignment_failure(&dir, &failure).unwrap(); + + let run = prepare_against_record(&dir, false, at(FAILED_AT + HOUR), || { + bail!("backend download failed") + }); + + assert_eq!( + run.result.unwrap_err().to_string(), + "backend download failed" + ); + assert_eq!(run.align_calls, 0); + assert_eq!(run.installs, [false]); + assert!( + run.notices.is_empty(), + "the notice says the pinned backend is in use, which it is not: {:?}", + run.notices + ); + assert_eq!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Recorded(failure) + ); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn the_opt_out_leaves_a_record_alone() { + let (dir, _) = runtime_dir_with_packaged_pin("record-opt-out"); + write_alignment_failure(&dir, &recorded_failure(RECORD_TARGET, FAILED_AT)).unwrap(); + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + let before = fs::read(&record_path).unwrap(); + + let run = prepare_against_record(&dir, true, at(FAILED_AT + HOUR), || Ok(())); + + assert_eq!(run.result.unwrap(), None); + assert_eq!(run.align_calls, 0); + assert_eq!(run.installs, [false]); + assert!( + run.notices.is_empty(), + "the opt-out prints its own line, not the record's: {:?}", + run.notices + ); + assert_eq!(fs::read(&record_path).unwrap(), before); + fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_verified_alignment_removes_the_record_of_an_earlier_failure() { + for tier2 in [false, true] { + let (dir, pin_file) = runtime_dir_with_packaged_pin("record-cleared"); + write_alignment_failure(&dir, &recorded_failure(RECORD_TARGET, FAILED_AT)).unwrap(); + let mut manifest = test_manifest(dir.clone()); + let mut attempts = 0; + let mut warnings = Vec::new(); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &pin_file, + RECORD_TARGET, + RECORD_PIN, + false, + |_manifest, _target, _force, _label| { + attempts += 1; + !tier2 || attempts == 2 + }, + |_manifest, _force| panic!("a verified alignment needs no fallback install"), + || Ok("b10952".to_owned()), + || at(FAILED_AT + 25 * HOUR), + &mut |message| warnings.push(message), + ); + + assert_eq!(result.unwrap(), Some(RECORD_TARGET.to_owned())); + assert_eq!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Absent, + "tier {} succeeded, so the record must go", + if tier2 { 2 } else { 1 } + ); + assert!(warnings.is_empty(), "{warnings:?}"); + fs::remove_dir_all(&dir).ok(); + } + } + + #[test] + fn a_record_that_cannot_be_written_warns_and_the_install_still_succeeds() { + let (dir, pin_file) = runtime_dir_with_packaged_pin("record-unwritable"); + // A non-empty directory where the record file goes: no write can replace it. + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::create_dir_all(record_path.join("occupied")).unwrap(); + let mut manifest = test_manifest(dir.clone()); + let mut installs = Vec::new(); + let mut warnings = Vec::new(); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &pin_file, + RECORD_TARGET, + RECORD_PIN, + false, + |_manifest, _target, _force, _label| false, + |_manifest, force_reinstall| { + installs.push(force_reinstall); + Ok(()) + }, + || Ok("b10952".to_owned()), + || at(FAILED_AT), + &mut |message| warnings.push(message), + ); + + assert_eq!(result.unwrap(), None, "the install itself succeeded"); + assert_eq!(installs, [true]); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + warnings[0].contains("could not record") + && warnings[0].contains("the next install tries it again"), + "{warnings:?}" + ); + assert!( + !matches!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Recorded(_) + ), + "nothing was recorded, so the warning's claim holds" + ); + assert_eq!( + read_backend_versions_therock_version(&pin_file).as_deref(), + Some(RECORD_PIN) + ); + fs::remove_dir_all(&dir).ok(); + } } diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index b6617fc215..72b70a01d4 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -967,6 +967,76 @@ mod tests { ); } + /// The recovery the CLI names when a recorded alignment failure skips the + /// alignment is `--reinstall`. That only holds if a reinstall removes the + /// record, which it does because the record lives inside the runtime tree a + /// reinstall deletes and re-extracts. A plain install of the same version + /// keeps the tree, record included, which is what lets the record skip the + /// alignment at all. + #[test] + fn reinstall_removes_the_record_of_a_failed_alignment() { + use crate::alignment_record::{ + AlignmentFailure, StoredAlignmentFailure, read_alignment_failure, + write_alignment_failure, + }; + let env_root = tempfile::tempdir().unwrap(); + let paths = AppPaths { + config_dir: env_root.path().join("config"), + data_dir: env_root.path().join("data"), + cache_dir: env_root.path().join("cache"), + }; + let version = "11.5.1"; + let archive_dir = tempfile::tempdir().unwrap(); + let archive = archive_dir.path().join("embeddable"); + write_embeddable_with_pins( + &archive, + r#"{"therock":{"version":"7.13.0"},"llamacpp":{"rocm-stable":"b9752"}}"#, + ); + let sha256 = format!("{:x}", Sha256::digest(fs::read(&archive).unwrap())); + let install = |reinstall: bool| { + prepare_embeddable_with( + &paths, + Some(env_root.path()), + reinstall, + version, + &sha256, + |_, destination| { + fs::copy(&archive, destination)?; + Ok(()) + }, + ) + .unwrap() + }; + let runtime_dir = install(false).runtime_dir; + let failure = AlignmentFailure { + target_rocm_version: "7.99.0".to_owned(), + pinned_rocm_version: "7.13.0".to_owned(), + pinned_llamacpp_tag: Some("b9752".to_owned()), + latest_llamacpp_tag: None, + lemonade_version: version.to_owned(), + failed_at_unix_secs: 1_800_000_000, + }; + write_alignment_failure(&runtime_dir, &failure).unwrap(); + + install(false); + assert_eq!( + read_alignment_failure(&runtime_dir), + StoredAlignmentFailure::Recorded(failure), + "a plain install keeps the tree and the record in it" + ); + + install(true); + assert_eq!( + read_alignment_failure(&runtime_dir), + StoredAlignmentFailure::Absent, + "--reinstall must remove the record, or the retry it advertises does not happen" + ); + assert!( + lemond_path_in(&runtime_dir).is_file(), + "the tree was re-extracted" + ); + } + /// Writes an embeddable archive in the host's format holding the two /// binaries and a packaged `resources/backend_versions.json`. fn write_embeddable_with_pins(archive: &Path, pins: &str) { diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 78ea6c164c..f9e306b5a2 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -18,6 +18,7 @@ use std::io::Read; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +mod alignment_record; mod archive_cache; mod archive_extract; /// Shared archive generator and filesystem oracle for the extraction property diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 24c61ea166..e1144d8ca2 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -718,3 +718,30 @@ Feature: Runtime configuration Given a machine with an AMD GPU When the user dry-runs a release SDK install pinned to an earlier ROCm 7 release Then the SDK preview reports canonical release provenance + + # ROCMAI-437. When Lemonade's ROCm backend cannot be aligned to the active ROCm + # version, the install reverts to the pinned backend and force-reinstalls it. + # The revert leaves the pin different from the active version, so before the + # failure was remembered every later install (and every `rocm serve` that + # installs Lemonade) ran both tiers' downloads and the forced reinstall again. + # The first install here is the failure; the second is the one under test. + # The failure is remembered for 24 hours; the unit tests in + # engines/lemonade/src/backend_alignment.rs cover expiry, another ROCm + # version, an unreadable record and the opt-out. + # + # Technical details: the version the alignment targets is overridden to + # 7.99.0 through the `rocm/e2e-test-hooks` variable + # ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION, so every tier fails on every lane + # without planting a runtime in the shared runtime tree. The Given's reinstall + # is also what the notice tells the user to run, so its alignment attempt shows + # the advice works. Same lane as runtime-16 and runtime-17: a real managed SDK + # and real backend downloads. `@requires-bare-metal` because under WSL2 + # Lemonade selects its Vulkan backend and never attempts the alignment. + @id:runtime-lemonade-backend-alignment-failure-remembered @requires-gpu @requires-bare-metal @requires-engine:lemonade @nightly + Scenario: runtime-33 - A Lemonade backend that could not be aligned is not retried by the next install + Given a managed runtime is active + And the active ROCm version is one Lemonade publishes no backend build for + And reinstalling Lemonade has already failed to align its backend to that version + When the user installs the Lemonade engine + Then the install reuses Lemonade's pinned backend without trying the alignment again + And the user is told the alignment failed and how to try it again diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 10a98f0295..b11e1065ff 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -9,6 +9,12 @@ //! planted as plain files in the layout the engine writes, so no scenario //! downloads anything; the refusal scenario reaches the engine's free-space //! check through the `rocm/e2e-test-hooks` override instead of a full drive. +//! +//! Also the steps of `runtime_setup.feature`'s runtime-33: a Lemonade backend +//! that could not be aligned to the active ROCm version is not tried again by +//! the next install. That one is not planted: it runs real installs on a GPU +//! lane, and only the ROCm version the alignment targets comes from an +//! `e2e-test-hooks` override. use std::path::PathBuf; @@ -400,3 +406,109 @@ async fn assert_lemonade_record_cleared(world: &mut E2eWorld) { "the removed copy must not be listed any more:\n{report}" ); } + +/// A ROCm version Lemonade publishes no backend build for, so every alignment +/// tier fails the same way on every lane. It reaches the engine through the +/// `rocm/e2e-test-hooks` override of the version the alignment targets, rather +/// than a planted runtime record, because the runtime tree this scenario installs +/// into is shared with every other scenario on the runner. +const UNPUBLISHED_ROCM_VERSION: &str = "7.99.0"; + +/// The override applies to one command only (`run_rocm_with_scenario_env` +/// consumes it), so each step that runs `rocm` sets it again first. +fn target_unpublished_rocm_version(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION", + UNPUBLISHED_ROCM_VERSION.into(), + )); +} + +const ALIGNMENT_FAILED_LINE: &str = "could not align Lemonade's ROCm backend to"; + +#[given("the active ROCm version is one Lemonade publishes no backend build for")] +async fn active_rocm_version_has_no_lemonade_build(world: &mut E2eWorld) { + target_unpublished_rocm_version(world); +} + +/// `--reinstall` deletes and re-extracts the runtime tree, so this attempt starts +/// from Lemonade's packaged pin with no record of an earlier failure, whatever an +/// earlier run on this runner left. It is also the command the later notice +/// names for trying again, so this step running the alignment is the evidence +/// that the advice works. +#[given("reinstalling Lemonade has already failed to align its backend to that version")] +async fn lemonade_reinstall_failed_alignment(world: &mut E2eWorld) { + let args = ["engines", "install", "lemonade", "--reinstall"]; + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &args); + assert!( + rc == 0, + "{}", + e2e_cucumber::cli_failure_report(&args, rc, &stdout, &stderr) + ); + // Absent when the binary lacks `rocm/e2e-test-hooks` (the real SDK version is + // targeted instead), or on a host where Lemonade selects its Vulkan backend + // and the alignment is never attempted. + assert!( + stderr.contains(&format!( + "{ALIGNMENT_FAILED_LINE} {UNPUBLISHED_ROCM_VERSION}" + )), + "the reinstall did not try and fail to align the backend to \ + {UNPUBLISHED_ROCM_VERSION}; was `rocm` built with `rocm/e2e-test-hooks`, and does \ + Lemonade select its ROCm backend here?\n{}", + e2e_cucumber::cli_failure_report(&args, rc, &stdout, &stderr) + ); + target_unpublished_rocm_version(world); +} + +/// No alignment tier ran and nothing was force-reinstalled: the install reused +/// the backend already in place, and the pin still names the packaged version. +#[then("the install reuses Lemonade's pinned backend without trying the alignment again")] +async fn assert_pinned_backend_reused(world: &mut E2eWorld) { + let output = combined_output(world); + assert_eq!(world.cli_rc, Some(0), "the install failed:\n{output}"); + for attempt in [ + ALIGNMENT_FAILED_LINE, + "could not install its pinned llama.cpp build", + "Aligned Lemonade's ROCm backend", + "Installing Lemonade llamacpp:", + ] { + assert!( + !output.contains(attempt), + "the install tried the alignment or reinstalled the backend again \ + (`{attempt}`):\n{output}" + ); + } + assert!( + output.contains("Using installed Lemonade llamacpp:rocm backend."), + "the install did not reuse the installed ROCm backend:\n{output}" + ); + let env_path = stdout(world) + .lines() + .find_map(|line| line.trim().strip_prefix("env_path: ")) + .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); + let pin_file = std::path::Path::new(env_path).join("resources/backend_versions.json"); + let pins: serde_json::Value = serde_json::from_slice( + &std::fs::read(&pin_file) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", pin_file.display())), + ) + .unwrap_or_else(|error| panic!("failed to parse {}: {error}", pin_file.display())); + assert_ne!( + pins["therock"]["version"].as_str(), + Some(UNPUBLISHED_ROCM_VERSION), + "the pin names the version that could not be aligned to:\n{pins}" + ); +} + +#[then("the user is told the alignment failed and how to try it again")] +async fn assert_alignment_retry_advice(world: &mut E2eWorld) { + let output = combined_output(world); + for expected in [ + format!("could not be aligned to ROCm {UNPUBLISHED_ROCM_VERSION}"), + "Run `rocm engines install lemonade --reinstall` to try again now".to_owned(), + "the alignment is tried again in".to_owned(), + ] { + assert!( + output.contains(&expected), + "expected `{expected}` in the install output:\n{output}" + ); + } +} From fd5954f232677d5015042f34fb490929fa371b26 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 15:06:21 +0200 Subject: [PATCH 2/6] fix(lemonade): address review of the alignment failure record (ROCMAI-437) An unreadable record of a failed alignment is now removed, with a warning that claims only that: whether the alignment then runs is decided later by the backend probe. When it cannot be removed either, the warning carries the removal's own error and names the command that removes it. The skip notice takes its window from the retry constant the decision uses, rounds up from the full time left so it never names a moment before the record expires, and no longer implies that anything retries on its own. Tests now cover every install that returns before the record is consulted, the removal failures after an unreadable record and after a verified alignment, sub-second rounding, and the record file name the e2e steps hard-code. The docs and the runtime-33 assertions are made exact: the scenario now checks the record and the reverted pin left by the failed attempt and that the next install keeps both. Signed-off-by: Roman Inflianskas --- docs/engine-plugins.md | 39 ++- docs/manual-testing.md | 2 +- docs/testing.md | 36 ++- engines/lemonade/src/alignment_record.rs | 102 +++++- engines/lemonade/src/backend_alignment.rs | 295 ++++++++++++++++-- .../features/runtime_setup.feature | 9 +- tests/e2e-cucumber/tests/e2e.rs | 5 + .../tests/e2e/lemonade_install_steps.rs | 87 ++++-- 8 files changed, 481 insertions(+), 94 deletions(-) diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index 48f3cb37d9..daa492c036 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -68,24 +68,31 @@ being kept. A failed alignment is remembered so that it is not repeated on every install. The revert puts back a pin that still differs from the active ROCm version, so -without that every later `rocm engines install lemonade`, and every `rocm -serve` that installs Lemonade, would download both builds and force-reinstall -the pinned backend again. When the revert's install succeeds, the attempt (the +without that every later `rocm engines install lemonade`, and every `rocm serve` +that installs Lemonade, would download both builds and force-reinstall the +pinned backend again. When the revert's install succeeds, the attempt (the active ROCm version, the pins it started from, the llama.cpp builds tried and the Lemonade version) is recorded in `.rocm-cli-alignment-failed.json` in the -Lemonade runtime tree. For the next 24 hours an install for the same active -ROCm version, pin and Lemonade version does not attempt the alignment: it -installs the backend already pinned without forcing a reinstall, so a backend -already installed is reused, and prints a line saying the alignment could not -be done, when it failed, and when it is tried again. `rocm engines install -lemonade --reinstall` tries it again at once, because it deletes and -re-extracts the runtime tree, record included. A different active ROCm version, -a new Lemonade version (which also re-extracts the tree), or a record older than -24 hours runs the alignment as before, and an alignment that succeeds removes -the record. A record that cannot be read is ignored with a warning, and one -that cannot be written leaves the next install to try again, also with a -warning. Setting `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` neither reads nor -changes the record. +Lemonade runtime tree. For the next 24 hours an install for the same active ROCm +version, pin and Lemonade version does not attempt the alignment: it installs +the backend already pinned without forcing a reinstall, so a backend already +installed is reused, and prints a line saying the alignment could not be done, +when it failed, and how many hours from now an install tries it again (nothing +retries on its own). `rocm engines install lemonade --reinstall` tries it again +at once, because it deletes and re-extracts the runtime tree, record included. A +different active ROCm version, a new Lemonade version (which also re-extracts +the tree), or a record 24 hours old or older (or dated in the future) runs the +alignment as before, and an alignment that succeeds removes the record. The record is consulted only when an +install could align: not with the opt-out set, on an unsupported platform, with +no active ROCm version or an unreadable pin, or when the pin already equals the +active ROCm version, all of which leave it untouched and say nothing about it. +Past those checks, a record that cannot be read is removed with a warning, +before the backend probe decides whether to align; if it cannot be removed +either, the warning says so and names `rocm engines install lemonade +--reinstall`, which removes it. A record that cannot be written leaves the next +install to try again, also with a warning. Setting +`ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` neither reads nor changes the +record. A pin an earlier alignment kept can go stale: the llama.cpp or TheRock build it names may later be withdrawn upstream. When the pin already equals the active diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 396c02cafd..0baed00c62 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -467,7 +467,7 @@ then `rocm engines install lemonade`. Expected result: reinstall the backend (`Using installed Lemonade llamacpp:rocm backend.`). It prints `Lemonade's ROCm backend could not be aligned to ROCm 7.99.0 less than an hour ago; ...`, naming `rocm engines install lemonade --reinstall` and - when the alignment is tried again, and exits 0. + that an install run 24 hours from now or later tries it again, and exits 0. - Running the first command again attempts the alignment again (the warning is back), because `--reinstall` removed the record with the runtime tree. diff --git a/docs/testing.md b/docs/testing.md index 505e0d99a0..f328326d9b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2400,21 +2400,31 @@ cargo test -p rocm-engine-lemonade a_failed_alignment_is_not_repeated cargo test -p rocm-engine-lemonade a_skipped_alignment_whose_install_fails ``` -The tests are in [alignment_record.rs](../engines/lemonade/src/alignment_record.rs) -and [backend_alignment.rs](../engines/lemonade/src/backend_alignment.rs), one per -state of the record: none, one that skips the alignment -(the regression test: the second install calls no alignment tier and installs -without forcing, and its notice is asserted with the pin file), one for another -ROCm version, one 24 hours old, an unreadable one, one that cannot be written, -one removed by a later successful alignment, one removed by `--reinstall` +The tests are in +[alignment_record.rs](../engines/lemonade/src/alignment_record.rs) and +[backend_alignment.rs](../engines/lemonade/src/backend_alignment.rs), one per +state of the record: none, one that skips the alignment (the regression test: +the second install calls no alignment tier and installs without forcing, and its +notice is asserted with the pin file), one for another ROCm version, one 24 +hours old, an unreadable one, one that cannot be written, one that cannot be +removed after a later successful alignment, one removed by a later successful +alignment, one removed by `--reinstall` (`reinstall_removes_the_record_of_a_failed_alignment` in -[install.rs](../engines/lemonade/src/install.rs)), one the -opt-out leaves alone, and a skipped alignment whose install fails (the error is -reported and no notice is printed). The scenario +[install.rs](../engines/lemonade/src/install.rs)), one the opt-out leaves alone, +and a skipped alignment whose install fails (the error is reported and no notice +is printed). The unreadable record has a row for each place it can be met: every +install that returns before the record is consulted (the opt-out, an unsupported +platform, no active ROCm version or a failed lookup of it, an unreadable pin, +and a pin that already equals the active ROCm version) leaves it as it is and +says nothing; one that gets past those checks removes it with a warning, whether +the alignment then runs, the backend probe fails or the probe selects Vulkan; +and when it cannot be removed, the warning carries the removal's own error and +names `rocm engines install lemonade --reinstall`. The scenario `runtime-lemonade-backend-alignment-failure-remembered` (runtime-33 in -[runtime_setup.feature](../tests/e2e-cucumber/features/runtime_setup.feature)) runs the two installs for real on the -`@nightly @requires-gpu` Lemonade lanes (not under WSL2); it targets ROCm 7.99.0 -through `ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION`, compiled only into the +[runtime_setup.feature](../tests/e2e-cucumber/features/runtime_setup.feature)) +runs the two installs for real on the `@nightly @requires-gpu` Lemonade lanes +(not under WSL2); it targets ROCm 7.99.0 through +`ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION`, compiled only into the `rocm/e2e-test-hooks` build. Engine recipe adapter contract focused tests: diff --git a/engines/lemonade/src/alignment_record.rs b/engines/lemonade/src/alignment_record.rs index 90280bc85c..5d197f1725 100644 --- a/engines/lemonade/src/alignment_record.rs +++ b/engines/lemonade/src/alignment_record.rs @@ -132,18 +132,21 @@ impl AlignmentFailure { /// The line printed when this record skipped the alignment and the pinned /// backend was installed instead. Built from the record and the same `now` /// [`Self::skips_alignment`] was asked with, so the times it names are the - /// ones the decision used. + /// ones the decision used. Nothing retries on its own: only an install that + /// runs once the record has expired attempts the alignment, so the line names + /// the earliest such install rather than a time something happens. That time + /// comes from [`ALIGNMENT_RETRY_AFTER`], the same window the decision uses. pub(crate) fn skip_notice(&self, now: SystemTime) -> String { let age = self.age(now).unwrap_or_default(); let remaining = ALIGNMENT_RETRY_AFTER.saturating_sub(age); format!( "Lemonade's ROCm backend could not be aligned to ROCm {target} {ago}; using the \ backend pinned for ROCm {pinned} instead. Run `{RETRY_ALIGNMENT_COMMAND}` to try \ - again now, or the alignment is tried again {when}.", + again now; otherwise an install run {when} or later tries it again.", target = self.target_rocm_version, ago = describe_ago(age), pinned = self.pinned_rocm_version, - when = describe_in(remaining), + when = describe_from_now(remaining), ) } } @@ -159,15 +162,16 @@ fn describe_ago(age: Duration) -> String { } } -/// "in less than an hour", "in 1 hour", "in 5 hours": whole hours, rounded up -/// so it never names a time before the record expires. -fn describe_in(remaining: Duration) -> String { - if remaining.as_secs() < HOUR { - return "in less than an hour".to_owned(); - } - match remaining.as_secs().div_ceil(HOUR) { - 1 => "in 1 hour".to_owned(), - hours => format!("in {hours} hours"), +/// "1 hour from now", "5 hours from now": whole hours, rounded up from the full +/// duration (a part of a second counts) so it never names a time before the +/// record expires. Nothing left, which [`AlignmentFailure::skip_notice`] never +/// passes since a record with no time left skips nothing, still reads as +/// "1 hour from now" rather than "0 hours". +fn describe_from_now(remaining: Duration) -> String { + const HOUR_NANOS: u128 = HOUR as u128 * 1_000_000_000; + match remaining.as_nanos().div_ceil(HOUR_NANOS).max(1) { + 1 => "1 hour from now".to_owned(), + hours => format!("{hours} hours from now"), } } @@ -176,6 +180,15 @@ mod tests { use super::*; use crate::test_support::scratch_dir; + /// `ALIGNMENT_FAILURE_RECORD` in the black-box e2e steps + /// (tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs, used by + /// runtime-33) hard-codes this name to find the record on disk; renaming it + /// here means changing it there too. + #[test] + fn the_record_file_name_matches_the_one_the_e2e_steps_hard_code() { + assert_eq!(ALIGNMENT_FAILURE_RECORD, ".rocm-cli-alignment-failed.json"); + } + fn record(failed_at_unix_secs: u64) -> AlignmentFailure { AlignmentFailure { target_rocm_version: "7.99.0".to_owned(), @@ -272,19 +285,74 @@ mod tests { fresh.contains("`rocm engines install lemonade --reinstall`"), "{fresh}" ); - assert!(fresh.contains("tried again in 24 hours"), "{fresh}"); + // The time named comes from the window the decision uses, so the line + // cannot name a different one than `skips_alignment` applies. + let window_hours = ALIGNMENT_RETRY_AFTER.as_secs() / HOUR; + assert!( + fresh.ends_with(&format!( + "to try again now; otherwise an install run {window_hours} hours from now or \ + later tries it again." + )), + "{fresh}" + ); + // The docs (docs/engine-plugins.md, docs/manual-testing.md) name 24 hours. + assert_eq!(window_hours, 24, "update the docs that name the window"); let later = failure.skip_notice(at(FAILED_AT + 5 * HOUR + 1)); assert!(later.contains("5 hours ago"), "{later}"); - assert!(later.contains("tried again in 19 hours"), "{later}"); + assert!( + later.contains("an install run 19 hours from now or later"), + "{later}" + ); let one = failure.skip_notice(at(FAILED_AT + HOUR)); assert!(one.contains("1 hour ago"), "{one}"); - assert!(one.contains("in 23 hours"), "{one}"); + assert!(one.contains("run 23 hours from now"), "{one}"); let last = failure.skip_notice(at(FAILED_AT + 23 * HOUR + 1)); - assert!(last.contains("in less than an hour"), "{last}"); + assert!(last.contains("run 1 hour from now or later"), "{last}"); let penultimate = failure.skip_notice(at(FAILED_AT + 23 * HOUR)); - assert!(penultimate.contains("in 1 hour"), "{penultimate}"); + assert!( + penultimate.contains("run 1 hour from now or later"), + "{penultimate}" + ); + // The last second of the window still names a time after it. + let final_second = failure.skip_notice(at(FAILED_AT + 24 * HOUR - 1)); + assert!( + final_second.contains("run 1 hour from now or later"), + "{final_second}" + ); + } + + /// `age` keeps the clock's sub-second part (the failure time is whole + /// seconds), so the time left is rarely whole seconds and a part of a second + /// must round up rather than be dropped. + #[test] + fn the_time_left_rounds_up_from_its_sub_second_part() { + let failure = record(FAILED_AT); + let window = ALIGNMENT_RETRY_AFTER.as_secs(); + + // 3,600.7 s left: "1 hour" would name a time 0.7 s before expiry. + let over_an_hour = + failure.skip_notice(at(FAILED_AT + window - HOUR - 1) + Duration::from_millis(300)); + assert!( + over_an_hour.contains("run 2 hours from now or later"), + "{over_an_hour}" + ); + + // 0.3 s left. + let last_moment = + failure.skip_notice(at(FAILED_AT + window - 1) + Duration::from_millis(700)); + assert!( + last_moment.contains("run 1 hour from now or later"), + "{last_moment}" + ); + + // No time left at all reads as the shortest wait, never "0 hours". + assert_eq!(describe_from_now(Duration::ZERO), "1 hour from now"); + assert_eq!( + describe_from_now(Duration::from_nanos(1)), + "1 hour from now" + ); } } diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index bb9d0592f7..e316beea62 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -17,8 +17,8 @@ use std::time::{Duration, SystemTime}; use crate::ROCM_BACKEND_NAME; use crate::alignment_record::{ - AlignmentFailure, StoredAlignmentFailure, clear_alignment_failure, read_alignment_failure, - unix_secs, write_alignment_failure, + AlignmentFailure, RETRY_ALIGNMENT_COMMAND, StoredAlignmentFailure, clear_alignment_failure, + read_alignment_failure, unix_secs, write_alignment_failure, }; use crate::install::{LemonadeInstallManifest, PinFileBackup}; use crate::llamacpp_backend::{ @@ -235,10 +235,28 @@ fn prepare_llamacpp_backend_for_active_rocm_impl( return Ok(None); } } - StoredAlignmentFailure::Unreadable(reason) => notice(format!( - "Warning: ignoring the record of an earlier failed Lemonade backend alignment \ - ({reason}); trying the alignment again." - )), + // Removed rather than only skipped: left in place it would print this + // warning on every install, and whether the alignment then runs is decided + // below (the probe can still skip it), so the warning claims neither. + // Accepted risk: a transient read failure (e.g. a sharing violation during a + // concurrent install's rename) deletes a valid record, costing one extra + // alignment attempt. + StoredAlignmentFailure::Unreadable(reason) => { + notice(match clear_alignment_failure(&manifest.runtime_dir) { + Ok(()) => format!( + "Warning: removed an unreadable record of an earlier failed Lemonade \ + backend alignment ({reason})." + ), + // `reason` and `error` both name the record's path; `error`'s + // root cause is the part `reason` does not already say. + Err(error) => format!( + "Warning: the record of an earlier failed Lemonade backend alignment is \ + unreadable ({reason}) and could not be removed ({}); \ + `{RETRY_ALIGNMENT_COMMAND}` removes it.", + error.root_cause() + ), + }); + } StoredAlignmentFailure::Absent => {} } @@ -311,8 +329,15 @@ fn prepare_llamacpp_backend_for_active_rocm_impl( /// the shared tree would reintroduce the orphan problem above. /// /// Scope: only the branch where the kept pin equals the active ROCm version calls -/// this. Other `fallback_install` call sites (no active SDK, an unreadable pin, a -/// failed backend probe) install what the file pins and do not restore. +/// this. Every other `fallback_install` call in +/// [`prepare_llamacpp_backend_for_active_rocm_impl`] installs what the file pins +/// and does not restore: the alignment opt-out, an unsupported platform, no active +/// SDK version, an unreadable pin, a recorded failure that skips the alignment +/// ([`crate::alignment_record`]), a failed backend probe, and a probe that selects a +/// backend other than ROCm (Vulkan, or none). The ones inside +/// [`align_llamacpp_backend_to_version`] (its opt-out, a pin that cannot be written, +/// and the revert, which puts back the pin the alignment started from rather than +/// the packaged one) do not call `restore_packaged_pins` either. /// /// What is reported when it does not recover: /// - a failed restore reports the original install error, which is the one the @@ -449,8 +474,8 @@ fn align_llamacpp_backend_to_version( // paired with the active ROCm version, forcing a real reinstall attempt // regardless of whatever Tier 1 left on disk. let pinned_tag = read_backend_versions_llamacpp_tag(backend_versions_path); - let latest_tag = latest_tag(); - let latest_tag_applied = match &latest_tag { + let latest_tag_lookup = latest_tag(); + let latest_tag_applied = match &latest_tag_lookup { Ok(latest_tag) if pinned_tag.as_deref() != Some(latest_tag.as_str()) => { match write_backend_versions_llamacpp_tag(backend_versions_path, latest_tag) { Ok(()) => true, @@ -520,7 +545,7 @@ fn align_llamacpp_backend_to_version( target_rocm_version: target_version.to_owned(), pinned_rocm_version: pinned_version.to_owned(), pinned_llamacpp_tag: pinned_tag, - latest_llamacpp_tag: latest_tag.ok(), + latest_llamacpp_tag: latest_tag_lookup.ok(), lemonade_version: manifest.version.clone(), failed_at_unix_secs: unix_secs(now()), }; @@ -542,9 +567,21 @@ fn align_llamacpp_backend_to_version( } /// Remove the record of an earlier failed alignment once an alignment has been -/// verified, so it cannot skip a later one. Best-effort: a record left behind is -/// not read while the pin equals the version it was aligned to, which is what a -/// verified alignment leaves. +/// verified, so it cannot skip a later one. Best-effort, with a warning: a record +/// left behind is not read while the pin equals the version it was aligned to, +/// which is what a verified alignment leaves. It is read by any later install +/// whose pin differs from the active ROCm version, and skips that install's +/// alignment when its target, pin and Lemonade version all match the attempt and +/// it is under [`ALIGNMENT_RETRY_AFTER`] old. A record outlives a verified +/// alignment only when this removal failed (records are written only on a failed +/// alignment, and every verified one removes them here). Only then can a stale-pin +/// restore ([`install_pinned_backend_or_restore_packaged_pins`]), putting the +/// packaged pin back with the active ROCm version unchanged, meet a record of a +/// failed alignment from that pin to that version: it then skips the alignment, +/// with a notice naming a failure the later alignment superseded, until it +/// expires or `--reinstall` removes it. +/// +/// [`ALIGNMENT_RETRY_AFTER`]: crate::alignment_record::ALIGNMENT_RETRY_AFTER fn forget_alignment_failure(manifest: &LemonadeInstallManifest, warn: &mut impl FnMut(String)) { if let Err(error) = clear_alignment_failure(&manifest.runtime_dir) { warn(format!( @@ -2463,6 +2500,19 @@ mod tests { disabled: bool, now: SystemTime, install_result: fn() -> Result<()>, + ) -> PrepareRun { + prepare_against_record_with_probe(dir, disabled, now, install_result, || { + Ok(Some(ROCM_BACKEND_NAME.to_owned())) + }) + } + + /// [`prepare_against_record`] with the backend probe's answer scripted. + fn prepare_against_record_with_probe( + dir: &Path, + disabled: bool, + now: SystemTime, + install_result: fn() -> Result<()>, + probe: fn() -> Result>, ) -> PrepareRun { let mut manifest = test_manifest(dir.to_path_buf()); let mut align_calls = 0; @@ -2476,7 +2526,7 @@ mod tests { |message| notices.push(message), || Ok(Some(RECORD_TARGET.to_owned())), read_backend_versions_therock_version, - |_manifest| Ok(Some(ROCM_BACKEND_NAME.to_owned())), + |_manifest| probe(), |_manifest, force_reinstall| { installs.push(force_reinstall); install_result() @@ -2576,7 +2626,10 @@ mod tests { notice.contains("rocm engines install lemonade --reinstall"), "{notice}" ); - assert!(notice.contains("tried again in 23 hours"), "{notice}"); + assert!( + notice.contains("otherwise an install run 23 hours from now or later tries it again."), + "{notice}" + ); assert_eq!( read_backend_versions_therock_version(&pin_file).as_deref(), Some(RECORD_PIN), @@ -2644,14 +2697,147 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// The record is consulted only once an install could align. On every path + /// that returns before that, an unreadable record is left as it is and + /// nothing is said about it, as docs/engine-plugins.md states. #[test] - fn an_unreadable_record_is_ignored_with_one_warning_and_the_alignment_runs() { - let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt"); - fs::write( - dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD), - b"{ torn", - ) - .unwrap(); + fn an_unreadable_record_is_left_alone_and_unmentioned_by_every_install_that_cannot_align() { + type SdkLookup = fn() -> Result>; + type PinLookup = fn(&Path) -> Option; + let target: SdkLookup = || Ok(Some(RECORD_TARGET.to_owned())); + let packaged_pin: PinLookup = read_backend_versions_therock_version; + let rows: [(&str, bool, bool, SdkLookup, PinLookup); 6] = [ + ("opt-out", true, false, target, packaged_pin), + ("unsupported platform", false, true, target, packaged_pin), + ( + "no active ROCm version", + false, + false, + || Ok(None), + packaged_pin, + ), + ( + "active ROCm version lookup fails", + false, + false, + || Err(anyhow!("no runtime record")), + packaged_pin, + ), + ("unreadable pin", false, false, target, |_path| None), + ( + "pin equals the active ROCm version", + false, + false, + target, + |_path| Some(RECORD_TARGET.to_owned()), + ), + ]; + for (label, disabled, unsupported_platform, sdk, pin) in rows { + let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt-early-exit"); + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::write(&record_path, b"{ torn").unwrap(); + let mut manifest = test_manifest(dir.clone()); + let mut notices = Vec::new(); + let mut installs = Vec::new(); + + let result = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + disabled, + unsupported_platform, + || at(FAILED_AT), + |message| notices.push(message), + sdk, + pin, + |_manifest| panic!("{label}: the backend probe is past the record"), + |_manifest, force_reinstall| { + installs.push(force_reinstall); + Ok(()) + }, + |_manifest| panic!("{label}: the pinned install succeeded; nothing restores"), + |_manifest, _path, _target, _pinned, _disabled| { + panic!("{label}: this install cannot align") + }, + ); + + assert_eq!(result.unwrap(), None, "{label}"); + assert_eq!(installs, vec![false], "{label}"); + assert!(notices.is_empty(), "{label}: {notices:?}"); + assert_eq!( + fs::read(&record_path).unwrap(), + b"{ torn", + "{label}: the record is left as it was" + ); + fs::remove_dir_all(&dir).ok(); + } + } + + /// An unreadable record is removed whatever happens next, so the warning + /// says only that, and is true on each path below: the alignment runs, the + /// probe fails, or the probe selects Vulkan (both of which skip it). + #[test] + fn an_unreadable_record_is_removed_with_one_warning_on_every_path() { + type Probe = fn() -> Result>; + let probes: [(&str, Probe, usize, Vec); 3] = [ + ("rocm", || Ok(Some(ROCM_BACKEND_NAME.to_owned())), 1, vec![]), + ( + "probe error", + || Err(anyhow!("lemond failed to start")), + 0, + vec![false], + ), + ("vulkan", || Ok(Some("vulkan".to_owned())), 0, vec![false]), + ]; + for (label, probe, align_calls, installs) in probes { + let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt"); + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::write(&record_path, b"{ torn").unwrap(); + + let run = + prepare_against_record_with_probe(&dir, false, at(FAILED_AT), || Ok(()), probe); + + assert_eq!(run.result.unwrap(), None, "{label}"); + assert_eq!(run.align_calls, align_calls, "{label}"); + assert_eq!(run.installs, installs, "{label}"); + assert_eq!(run.notices.len(), 1, "{label}: {:?}", run.notices); + assert!( + run.notices[0].starts_with( + "Warning: removed an unreadable record of an earlier failed Lemonade \ + backend alignment" + ), + "{label}: {:?}", + run.notices + ); + assert!( + !record_path.exists(), + "{label}: the warning says the record was removed" + ); + + let next = + prepare_against_record_with_probe(&dir, false, at(FAILED_AT), || Ok(()), probe); + assert!( + next.notices.is_empty(), + "{label}: the warning does not recur: {:?}", + next.notices + ); + fs::remove_dir_all(&dir).ok(); + } + } + + /// When the unreadable record cannot be removed either, the warning says so + /// and names the command that does remove it (`--reinstall` deletes the + /// runtime tree; `reinstall_removes_the_record_of_a_failed_alignment` in + /// `install.rs` proves that), and the alignment still runs. + #[test] + fn an_unreadable_record_that_cannot_be_removed_names_the_command_that_removes_it() { + let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt-stuck"); + // A directory where the record file goes: reading a directory as a file + // fails, and so does removing it as one, since `remove_file` refuses any + // directory, empty or not. + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::create_dir_all(&record_path).unwrap(); + // What the platform says when that removal fails, so the warning can be held + // to carrying it on Linux and Windows alike. + let removal_error = fs::remove_file(&record_path).unwrap_err().to_string(); let run = prepare_against_record(&dir, false, at(FAILED_AT), || Ok(())); @@ -2662,11 +2848,27 @@ mod tests { ); assert_eq!(run.notices.len(), 1, "{:?}", run.notices); assert!( - run.notices[0].starts_with("Warning: ignoring the record") - && run.notices[0].contains("trying the alignment again"), + run.notices[0].starts_with( + "Warning: the record of an earlier failed Lemonade backend alignment is \ + unreadable (could not read " + ) && run.notices[0].contains("`rocm engines install lemonade --reinstall`"), "{:?}", run.notices ); + assert_eq!( + run.notices[0] + .matches(record_path.display().to_string().as_str()) + .count(), + 1, + "the path is named once: {:?}", + run.notices + ); + assert!( + run.notices[0].contains(&format!("and could not be removed ({removal_error}); ")), + "expected the removal's own error `{removal_error}`: {:?}", + run.notices + ); + assert!(record_path.is_dir(), "the warning says it was not removed"); fs::remove_dir_all(&dir).ok(); } @@ -2756,6 +2958,49 @@ mod tests { } } + #[test] + fn a_record_that_cannot_be_removed_after_a_verified_alignment_warns_and_keeps_the_alignment() { + let (dir, pin_file) = runtime_dir_with_packaged_pin("record-not-removable"); + // A directory where the record file goes: `remove_file` refuses any + // directory, empty or not. + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::create_dir_all(&record_path).unwrap(); + let mut manifest = test_manifest(dir.clone()); + let mut warnings = Vec::new(); + + let result = align_llamacpp_backend_to_version( + &mut manifest, + &pin_file, + RECORD_TARGET, + RECORD_PIN, + false, + |_manifest, _target, _force, _label| true, + |_manifest, _force| panic!("a verified alignment needs no fallback install"), + || panic!("tier 1 succeeded; tier 2's tag lookup must not run"), + || at(FAILED_AT), + &mut |message| warnings.push(message), + ); + + assert_eq!( + result.unwrap(), + Some(RECORD_TARGET.to_owned()), + "the alignment itself succeeded and is kept" + ); + assert_eq!( + read_backend_versions_therock_version(&pin_file).as_deref(), + Some(RECORD_TARGET) + ); + assert_eq!(warnings.len(), 1, "{warnings:?}"); + assert!( + warnings[0].starts_with( + "could not remove the record of an earlier failed Lemonade backend alignment" + ), + "{warnings:?}" + ); + assert!(record_path.is_dir(), "the warning says it was not removed"); + fs::remove_dir_all(&dir).ok(); + } + #[test] fn a_record_that_cannot_be_written_warns_and_the_install_still_succeeds() { let (dir, pin_file) = runtime_dir_with_packaged_pin("record-unwritable"); diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index e1144d8ca2..cb40d7d2e7 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -733,9 +733,12 @@ Feature: Runtime configuration # 7.99.0 through the `rocm/e2e-test-hooks` variable # ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION, so every tier fails on every lane # without planting a runtime in the shared runtime tree. The Given's reinstall - # is also what the notice tells the user to run, so its alignment attempt shows - # the advice works. Same lane as runtime-16 and runtime-17: a real managed SDK - # and real backend downloads. `@requires-bare-metal` because under WSL2 + # starts from a clean runtime tree; that `--reinstall` removes the record, as + # the notice says, is unit-tested by + # `reinstall_removes_the_record_of_a_failed_alignment` in + # engines/lemonade/src/install.rs, since on a fresh runner there is no record + # for this scenario's reinstall to remove. Same lane as runtime-16 and + # runtime-17: a real managed SDK and real backend downloads. `@requires-bare-metal` because under WSL2 # Lemonade selects its Vulkan backend and never attempts the alignment. @id:runtime-lemonade-backend-alignment-failure-remembered @requires-gpu @requires-bare-metal @requires-engine:lemonade @nightly Scenario: runtime-33 - A Lemonade backend that could not be aligned is not retried by the next install diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6e8db6c579..a1057bba4d 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -203,6 +203,10 @@ pub struct E2eWorld { /// field rather than borrowing `model_name`, which means a served model and /// has nothing to do with a user's error report. pub skill_symptom: Option, + /// The `therock.version` Lemonade's `backend_versions.json` held after + /// runtime-33's failed alignment had reverted it, so the install that follows + /// can be held to leaving exactly that pin in place. + pub lemonade_pin_after_failed_alignment: Option, /// The pasted error a diagnose-59 or diagnose-60 scenario hands to /// `rocm diagnose --symptom-file`. Its own field for the reason /// `skill_symptom` is. @@ -402,6 +406,7 @@ impl Default for E2eWorld { lifecycle: None, skill_reference: None, skill_symptom: None, + lemonade_pin_after_failed_alignment: None, symptom_paste: None, symptom_file_runs: Vec::new(), helper_identity: None, diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index b11e1065ff..e334400302 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -432,9 +432,14 @@ async fn active_rocm_version_has_no_lemonade_build(world: &mut E2eWorld) { /// `--reinstall` deletes and re-extracts the runtime tree, so this attempt starts /// from Lemonade's packaged pin with no record of an earlier failure, whatever an -/// earlier run on this runner left. It is also the command the later notice -/// names for trying again, so this step running the alignment is the evidence -/// that the advice works. +/// earlier run on this runner left. That `--reinstall` removes a record is not +/// shown here (there is none to remove on a fresh runner); it is +/// `reinstall_removes_the_record_of_a_failed_alignment` in the engine's +/// `install.rs`. +/// +/// Also checks the failure left the two things the next install depends on: the +/// record in the runtime tree, and the pin the revert restored, which it keeps +/// for the Then to compare against. #[given("reinstalling Lemonade has already failed to align its backend to that version")] async fn lemonade_reinstall_failed_alignment(world: &mut E2eWorld) { let args = ["engines", "install", "lemonade", "--reinstall"]; @@ -456,11 +461,56 @@ async fn lemonade_reinstall_failed_alignment(world: &mut E2eWorld) { Lemonade select its ROCm backend here?\n{}", e2e_cucumber::cli_failure_report(&args, rc, &stdout, &stderr) ); + let runtime_dir = install_env_path(&stdout); + assert!( + runtime_dir.join(ALIGNMENT_FAILURE_RECORD).is_file(), + "the failed alignment was not recorded in {}:\n{}", + runtime_dir.display(), + e2e_cucumber::cli_failure_report(&args, rc, &stdout, &stderr) + ); + let pin = pinned_rocm_version(&runtime_dir); + assert_ne!( + pin, UNPUBLISHED_ROCM_VERSION, + "the failed alignment left its target pinned instead of reverting it" + ); + world.lemonade_pin_after_failed_alignment = Some(pin); target_unpublished_rocm_version(world); } +/// The record a failed alignment leaves in the Lemonade runtime tree. The +/// engine's own name for it, repeated here because this crate is black-box; +/// `the_record_file_name_matches_the_one_the_e2e_steps_hard_code` in the engine's +/// `alignment_record.rs` holds the engine constant to this literal. +const ALIGNMENT_FAILURE_RECORD: &str = ".rocm-cli-alignment-failed.json"; + +/// The runtime tree an `engines install` reports as its `env_path:`. +fn install_env_path(stdout: &str) -> PathBuf { + stdout + .lines() + .find_map(|line| line.trim().strip_prefix("env_path: ")) + .map_or_else( + || panic!("no env_path in install output:\n{stdout}"), + PathBuf::from, + ) +} + +/// `therock.version` in the runtime tree's `resources/backend_versions.json`. +fn pinned_rocm_version(runtime_dir: &std::path::Path) -> String { + let pin_file = runtime_dir.join("resources/backend_versions.json"); + let pins: serde_json::Value = serde_json::from_slice( + &std::fs::read(&pin_file) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", pin_file.display())), + ) + .unwrap_or_else(|error| panic!("failed to parse {}: {error}", pin_file.display())); + pins["therock"]["version"] + .as_str() + .unwrap_or_else(|| panic!("no therock.version in {}:\n{pins}", pin_file.display())) + .to_owned() +} + /// No alignment tier ran and nothing was force-reinstalled: the install reused -/// the backend already in place, and the pin still names the packaged version. +/// the backend already in place, the pin is exactly the one the failed alignment +/// reverted to, and the record that skipped the alignment is still there. #[then("the install reuses Lemonade's pinned backend without trying the alignment again")] async fn assert_pinned_backend_reused(world: &mut E2eWorld) { let output = combined_output(world); @@ -481,20 +531,19 @@ async fn assert_pinned_backend_reused(world: &mut E2eWorld) { output.contains("Using installed Lemonade llamacpp:rocm backend."), "the install did not reuse the installed ROCm backend:\n{output}" ); - let env_path = stdout(world) - .lines() - .find_map(|line| line.trim().strip_prefix("env_path: ")) - .unwrap_or_else(|| panic!("no env_path in install output:\n{output}")); - let pin_file = std::path::Path::new(env_path).join("resources/backend_versions.json"); - let pins: serde_json::Value = serde_json::from_slice( - &std::fs::read(&pin_file) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", pin_file.display())), - ) - .unwrap_or_else(|error| panic!("failed to parse {}: {error}", pin_file.display())); - assert_ne!( - pins["therock"]["version"].as_str(), - Some(UNPUBLISHED_ROCM_VERSION), - "the pin names the version that could not be aligned to:\n{pins}" + let runtime_dir = install_env_path(stdout(world)); + let expected_pin = world + .lemonade_pin_after_failed_alignment + .as_deref() + .expect("no pin captured after the failed alignment - did the Given run?"); + assert_eq!( + pinned_rocm_version(&runtime_dir), + expected_pin, + "the install changed the pin the failed alignment had reverted to:\n{output}" + ); + assert!( + runtime_dir.join(ALIGNMENT_FAILURE_RECORD).is_file(), + "the record that skipped the alignment is gone:\n{output}" ); } @@ -504,7 +553,7 @@ async fn assert_alignment_retry_advice(world: &mut E2eWorld) { for expected in [ format!("could not be aligned to ROCm {UNPUBLISHED_ROCM_VERSION}"), "Run `rocm engines install lemonade --reinstall` to try again now".to_owned(), - "the alignment is tried again in".to_owned(), + "hours from now or later tries it again.".to_owned(), ] { assert!( output.contains(&expected), From f50653f2658dd867119a050add398ea6cdd0429a Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 19:23:20 +0200 Subject: [PATCH 3/6] test(lemonade): pin the alignment record's ordering and file name (ROCMAI-437) Pin the behaviour the review asked about rather than only the code: - a revert whose install fails writes no record, so the next install never reuses a backend that failed to install; - a record that applies and an unreadable record are both handled before the backend probe, which starts lemond, so a skipped install never pays for the probe and the unreadable record's warning comes first; - the engine test now reads the e2e steps' copy of the record file name, so a rename on either side fails on every lane instead of only on the nightly scenario. Also document the two llama.cpp tag fields as informational: the skip decision does not read them. Signed-off-by: Roman Inflianskas --- docs/testing.md | 6 +- engines/lemonade/src/alignment_record.rs | 21 +++-- engines/lemonade/src/backend_alignment.rs | 89 ++++++++++++++++++- .../tests/e2e/lemonade_install_steps.rs | 3 +- 4 files changed, 110 insertions(+), 9 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f328326d9b..afb1ef19f6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2419,7 +2419,11 @@ and a pin that already equals the active ROCm version) leaves it as it is and says nothing; one that gets past those checks removes it with a warning, whether the alignment then runs, the backend probe fails or the probe selects Vulkan; and when it cannot be removed, the warning carries the removal's own error and -names `rocm engines install lemonade --reinstall`. The scenario +names `rocm engines install lemonade --reinstall`. Both a record that applies +and an unreadable one are handled before the backend probe starts `lemond`; a +revert whose install fails writes no record; and +`the_record_file_name_matches_the_one_the_e2e_steps_hard_code` reads the e2e +steps' copy of the record's file name, so the two cannot drift. The scenario `runtime-lemonade-backend-alignment-failure-remembered` (runtime-33 in [runtime_setup.feature](../tests/e2e-cucumber/features/runtime_setup.feature)) runs the two installs for real on the `@nightly @requires-gpu` Lemonade lanes diff --git a/engines/lemonade/src/alignment_record.rs b/engines/lemonade/src/alignment_record.rs index 5d197f1725..9c2b8ba1f2 100644 --- a/engines/lemonade/src/alignment_record.rs +++ b/engines/lemonade/src/alignment_record.rs @@ -37,8 +37,12 @@ pub(crate) struct AlignmentFailure { /// the revert put back. pub(crate) pinned_rocm_version: String, /// The llama.cpp build tag pinned before the attempt (tier 1 tried it). + /// Informational, for a person reading the file: the skip decision + /// ([`Self::skips_alignment`]) does not read it. pub(crate) pinned_llamacpp_tag: Option, /// The newest llama.cpp build tag tier 2 looked up, when the lookup worked. + /// Informational, for a person reading the file: the skip decision does not + /// read it. pub(crate) latest_llamacpp_tag: Option, /// The Lemonade version whose runtime tree the attempt ran in. pub(crate) lemonade_version: String, @@ -180,13 +184,20 @@ mod tests { use super::*; use crate::test_support::scratch_dir; - /// `ALIGNMENT_FAILURE_RECORD` in the black-box e2e steps - /// (tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs, used by - /// runtime-33) hard-codes this name to find the record on disk; renaming it - /// here means changing it there too. + /// `ALIGNMENT_FAILURE_RECORD` in the black-box e2e steps (used by + /// runtime-33) hard-codes this name to find the record on disk. Read from that + /// file, so a rename on either side fails here, on every lane, rather than + /// only when the nightly scenario runs. The include is test-only, and no + /// workspace crate is packaged for publishing (`publish = false`). #[test] fn the_record_file_name_matches_the_one_the_e2e_steps_hard_code() { - assert_eq!(ALIGNMENT_FAILURE_RECORD, ".rocm-cli-alignment-failed.json"); + let steps = include_str!("../../../tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs"); + let definition = + format!("const ALIGNMENT_FAILURE_RECORD: &str = \"{ALIGNMENT_FAILURE_RECORD}\";"); + assert!( + steps.contains(&definition), + "lemonade_install_steps.rs no longer defines `{definition}`" + ); } fn record(failed_at_unix_secs: u64) -> AlignmentFailure { diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index e316beea62..45570d0156 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -2330,6 +2330,13 @@ mod tests { b"pre-existing", "the pre-existing backend must be restored, not left deleted" ); + // A failure is recorded only once the pinned backend is back in place; a + // record here would tell the next install to reuse a backend that failed + // to install. + assert!( + matches!(read_alignment_failure(&dir), StoredAlignmentFailure::Absent), + "no record is written when the revert's install fails" + ); fs::remove_dir_all(&dir).ok(); } @@ -2453,8 +2460,11 @@ mod tests { // the record's state matrix: no record, a record that applies, one for another // target, an expired one, an unreadable one, a record that cannot be written, // an alignment that later succeeds, the opt-out, and an install that fails - // after the record skipped the alignment. `--reinstall` removing the record is - // in `install.rs` (`reinstall_removes_the_record_of_a_failed_alignment`). + // after the record skipped the alignment. A revert whose install fails writes + // no record: that is asserted in + // `align_revert_restores_the_backend_dir_when_the_fallback_install_fails`. + // `--reinstall` removing the record is in `install.rs` + // (`reinstall_removes_the_record_of_a_failed_alignment`). const RECORD_TARGET: &str = "7.99.0"; const RECORD_PIN: &str = "7.13.0"; @@ -2654,6 +2664,81 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// An unreadable record is handled where any record is read, before the + /// backend probe: by the time the probe runs (the alignment can still go + /// ahead), the record is already gone and its warning already printed. + #[test] + fn an_unreadable_record_is_removed_and_reported_before_the_backend_probe_runs() { + let (dir, _) = runtime_dir_with_packaged_pin("record-corrupt-before-probe"); + let record_path = dir.join(crate::alignment_record::ALIGNMENT_FAILURE_RECORD); + fs::write(&record_path, b"{ torn").unwrap(); + let mut manifest = test_manifest(dir.clone()); + let notices = std::cell::RefCell::new(Vec::new()); + let mut probe_calls = 0; + let mut align_calls = 0; + + let result = prepare_llamacpp_backend_for_active_rocm_impl( + &mut manifest, + false, + false, + || at(FAILED_AT), + |message| notices.borrow_mut().push(message), + || Ok(Some(RECORD_TARGET.to_owned())), + read_backend_versions_therock_version, + |_manifest| { + probe_calls += 1; + assert!( + !record_path.exists(), + "the probe ran before the unreadable record was removed" + ); + assert_eq!( + notices.borrow().len(), + 1, + "the probe ran before the warning was printed" + ); + Ok(Some(ROCM_BACKEND_NAME.to_owned())) + }, + |_manifest, _force_reinstall| panic!("align owns every install on this path"), + |_manifest| panic!("a pin that differs from the target never restores"), + |_manifest, _path, _target, _pinned, _disabled| { + align_calls += 1; + Ok(None) + }, + ); + + assert_eq!(result.unwrap(), None); + assert_eq!(probe_calls, 1); + assert_eq!(align_calls, 1); + assert!( + notices.borrow()[0].starts_with("Warning: removed an unreadable record"), + "{:?}", + notices.borrow() + ); + fs::remove_dir_all(&dir).ok(); + } + + /// The record is checked before the backend probe, which starts `lemond`: an + /// install the record skips never pays for that probe. + #[test] + fn a_record_that_applies_skips_before_the_backend_probe_runs() { + let (dir, _) = runtime_dir_with_packaged_pin("record-skips-probe"); + write_alignment_failure(&dir, &recorded_failure(RECORD_TARGET, FAILED_AT)).unwrap(); + + let run = prepare_against_record_with_probe( + &dir, + false, + at(FAILED_AT + HOUR), + || Ok(()), + || panic!("the record skipped the alignment; the backend probe must not run"), + ); + + assert_eq!(run.result.unwrap(), None); + assert_eq!(run.align_calls, 0); + assert_eq!(run.installs, vec![false]); + assert_eq!(run.notices.len(), 1, "{:?}", run.notices); + fs::remove_dir_all(&dir).ok(); + } + #[test] fn a_record_for_another_rocm_version_does_not_skip_the_alignment() { let (dir, _) = runtime_dir_with_packaged_pin("record-other-target"); diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index e334400302..77bc8b5ac6 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -480,7 +480,8 @@ async fn lemonade_reinstall_failed_alignment(world: &mut E2eWorld) { /// The record a failed alignment leaves in the Lemonade runtime tree. The /// engine's own name for it, repeated here because this crate is black-box; /// `the_record_file_name_matches_the_one_the_e2e_steps_hard_code` in the engine's -/// `alignment_record.rs` holds the engine constant to this literal. +/// `alignment_record.rs` reads this definition and fails on every lane if it +/// stops matching the engine constant, so keep it on one line as written. const ALIGNMENT_FAILURE_RECORD: &str = ".rocm-cli-alignment-failed.json"; /// The runtime tree an `engines install` reports as its `env_path:`. From c05946206436a28d8605c32d94d6e41656f48695 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 19:49:55 +0200 Subject: [PATCH 4/6] test(lemonade): name the tests behind the pinned --reinstall advice (ROCMAI-437) The three assertions that pin the `rocm engines install lemonade --reinstall` advice now name the tests that prove it: `reinstall_removes_the_record_of_a_failed_alignment` shows that the reinstall removes the failure record, and `with_no_record_the_alignment_runs_as_before` shows that the alignment then runs again. This follows the repository's rule that pinned remediation text points at the test proving the behavior. Also re-wrap a docs paragraph without changing its wording. Signed-off-by: Roman Inflianskas --- docs/engine-plugins.md | 18 +++++++++--------- engines/lemonade/src/alignment_record.rs | 4 ++++ engines/lemonade/src/backend_alignment.rs | 3 +++ .../tests/e2e/lemonade_install_steps.rs | 5 +++++ 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index daa492c036..0a3b5d6846 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -82,15 +82,15 @@ retries on its own). `rocm engines install lemonade --reinstall` tries it again at once, because it deletes and re-extracts the runtime tree, record included. A different active ROCm version, a new Lemonade version (which also re-extracts the tree), or a record 24 hours old or older (or dated in the future) runs the -alignment as before, and an alignment that succeeds removes the record. The record is consulted only when an -install could align: not with the opt-out set, on an unsupported platform, with -no active ROCm version or an unreadable pin, or when the pin already equals the -active ROCm version, all of which leave it untouched and say nothing about it. -Past those checks, a record that cannot be read is removed with a warning, -before the backend probe decides whether to align; if it cannot be removed -either, the warning says so and names `rocm engines install lemonade ---reinstall`, which removes it. A record that cannot be written leaves the next -install to try again, also with a warning. Setting +alignment as before, and an alignment that succeeds removes the record. The +record is consulted only when an install could align: not with the opt-out set, +on an unsupported platform, with no active ROCm version or an unreadable pin, or +when the pin already equals the active ROCm version, all of which leave it +untouched and say nothing about it. Past those checks, a record that cannot be +read is removed with a warning, before the backend probe decides whether to +align; if it cannot be removed either, the warning says so and names `rocm +engines install lemonade --reinstall`, which removes it. A record that cannot be +written leaves the next install to try again, also with a warning. Setting `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` neither reads nor changes the record. diff --git a/engines/lemonade/src/alignment_record.rs b/engines/lemonade/src/alignment_record.rs index 9c2b8ba1f2..45b6aedf13 100644 --- a/engines/lemonade/src/alignment_record.rs +++ b/engines/lemonade/src/alignment_record.rs @@ -292,6 +292,10 @@ mod tests { "{fresh}" ); assert!(fresh.contains("pinned for ROCm 7.13.0"), "{fresh}"); + // Pins the advice only. That `--reinstall` removes the record is + // `reinstall_removes_the_record_of_a_failed_alignment` in `install.rs`, and + // that the alignment then runs is `with_no_record_the_alignment_runs_as_before` + // in `backend_alignment.rs`. assert!( fresh.contains("`rocm engines install lemonade --reinstall`"), "{fresh}" diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 45570d0156..7526b3d6db 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -2632,6 +2632,9 @@ mod tests { notice.contains("aligned to ROCm 7.99.0 1 hour ago"), "{notice}" ); + // Pins the advice only. That `--reinstall` removes the record is + // `reinstall_removes_the_record_of_a_failed_alignment` in `install.rs`, and + // that the alignment then runs is `with_no_record_the_alignment_runs_as_before`. assert!( notice.contains("rocm engines install lemonade --reinstall"), "{notice}" diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index 77bc8b5ac6..75e3444612 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -548,6 +548,11 @@ async fn assert_pinned_backend_reused(world: &mut E2eWorld) { ); } +/// Pins the advice only; this scenario does not run the `--reinstall` it names. +/// That `--reinstall` removes the record is +/// `reinstall_removes_the_record_of_a_failed_alignment` in the engine's +/// `install.rs`, and that the alignment then runs is +/// `with_no_record_the_alignment_runs_as_before` in its `backend_alignment.rs`. #[then("the user is told the alignment failed and how to try it again")] async fn assert_alignment_retry_advice(world: &mut E2eWorld) { let output = combined_output(world); From f247212abdbba8be9ca39e2e366bc6cbc6c7a804 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 16 Nov 2026 20:48:59 +0200 Subject: [PATCH 5/6] dev: run the lemonade tests when the e2e steps they read change (ROCMAI-437) The alignment-record test in rocm-engine-lemonade include_str!s the e2e Lemonade install steps to check that the record file name they hard-code still matches the engine's. The pre-push hook selects crates by directory, so a push touching only that steps file never ran the engine's tests. Add the file to CROSS_CRATE_READS so it also selects rocm-engine-lemonade, and document the entry beside the others in docs/testing.md. This also satisfies the guard test every_cross_crate_read_is_covered_by_the_widening_rule, which fails on an uncovered cross-crate read. Signed-off-by: Roman Inflianskas --- docs/testing.md | 6 ++++-- xtask/src/prepush.rs | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index afb1ef19f6..610a0af509 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -992,8 +992,10 @@ pushed (`PRE_COMMIT_FROM_REF..PRE_COMMIT_TO_REF`, the commits the remote does no The directory-based selection cannot see a crate's tests reading a file from outside the crate, so the hook widens it: any Rust file also selects `xtask` (its contract tests scan every source file), `README.md`, `docs/` and `skills/` also select `rocm` and `xtask` (the advised-commands -test reads them and `rocm` compiles in the assistant skill), and the e2e PowerShell fixture also -selects `rocm-core` (its tests `include_str!` it). The table is `CROSS_CRATE_READS` in +test reads them and `rocm` compiles in the assistant skill), the e2e PowerShell fixture also +selects `rocm-core` (its tests `include_str!` it), and the e2e Lemonade install steps also select +`rocm-engine-lemonade` (its `alignment_record` tests `include_str!` them to check the record's file +name). The table is `CROSS_CRATE_READS` in [prepush.rs](../xtask/src/prepush.rs), and a unit test scans the workspace for `include_str!`, `include_bytes!` and manifest-relative reads that leave their crate and fails when one is not covered, so a new cross-crate read cannot be added without a decision. The scan has a limit: a read diff --git a/xtask/src/prepush.rs b/xtask/src/prepush.rs index 9543c92115..25a1076452 100644 --- a/xtask/src/prepush.rs +++ b/xtask/src/prepush.rs @@ -213,12 +213,19 @@ fn package_args<'a>(packages: Option>) -> Vec bool { From 008ba67bac4c5c4a497b5ee4d9c4e6722b62f84b Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 18:42:18 +0200 Subject: [PATCH 6/6] fix(lemonade): name the recorded failure when a skipped alignment's install fails (ROCMAI-437) After a recorded failure skips the backend alignment, a failing install of the pinned backend used to surface only the error beneath, leaving no hint that the alignment had been skipped or how to try it again. The error now says the alignment was skipped and why, and what `rocm engines install lemonade --reinstall` does: it re-extracts the runtime tree without the record, so it tries the alignment again and downloads the backend again. The line says only what that command does, not that it fixes the error beneath, which may be transient. The record is deliberately not keyed on the rocm-cli version: the engine cannot see the per-release version, so it expires after 24 hours and `--reinstall` clears it sooner. The gate list is stated once in docs/engine-plugins.md and docs/testing.md links to it; the e2e override comment describes which step re-arms it; the backend retry advice is a shared constant so the test can pin the full error chain against it. Signed-off-by: Roman Inflianskas --- README.md | 9 +- docs/architecture.md | 2 +- docs/engine-plugins.md | 45 ++++---- docs/manual-testing.md | 8 ++ docs/testing.md | 44 ++++---- engines/lemonade/src/alignment_record.rs | 30 ++++++ engines/lemonade/src/backend_alignment.rs | 101 +++++++++++++----- engines/lemonade/src/llamacpp_backend.rs | 10 +- .../features/runtime_setup.feature | 3 +- .../tests/e2e/lemonade_install_steps.rs | 6 +- 10 files changed, 186 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 787e2d47c1..0951f9a218 100644 --- a/README.md +++ b/README.md @@ -1089,9 +1089,12 @@ with the ROCm SDK in use, and rewrites the two pins it owns to do so. If installing an already-pinned backend fails, it restores those two pins from the cached Lemonade archive (it does not download), prints a warning, and installs once more. When the backend cannot be aligned to the active ROCm -version, the install keeps the backend Lemonade pins, and for the next 24 hours -later installs reuse it without trying the alignment again; -`rocm engines install lemonade --reinstall` tries it again sooner. See +version, the install keeps the backend Lemonade pins. For 24 hours after that, +an install with the same ROCm version, pin and Lemonade version reuses that +backend without trying the alignment again; if preparing it fails, +the error says the alignment was skipped and why. +`rocm engines install lemonade --reinstall` re-extracts Lemonade without that +record, so it tries the alignment again sooner. See [Lemonade backend alignment](https://github.com/ROCm/rocm-cli/blob/main/docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install) for the details and for `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, which turns all of this off. diff --git a/docs/architecture.md b/docs/architecture.md index 01b9864b61..f8cf8d73f5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -658,7 +658,7 @@ The Lemonade engine adapter (`rocm-engine-lemonade`): installs the embeddable Le | --- | --- | --- | | [`lib.rs`](../engines/lemonade/src/lib.rs) | Crate root: CLI and envelope dispatch, the service-lifecycle orchestration that calls across the modules, and the crate-root re-exports `apps/rocm` uses. | flat | | [`main.rs`](../engines/lemonade/src/main.rs) | The `rocm-engine-lemonade` binary's entry point; it only calls `rocm_engine_lemonade::run_cli()`. | flat | -| [`alignment_record.rs`](../engines/lemonade/src/alignment_record.rs) | The record, inside the runtime directory, that aligning the llama.cpp backend to a ROCm version failed: reading, writing and clearing it, whether it still skips the same alignment (24 hours), and the line printed when it does. | domain | +| [`alignment_record.rs`](../engines/lemonade/src/alignment_record.rs) | The record, inside the runtime directory, that aligning the llama.cpp backend to a ROCm version failed: reading, writing and clearing it, whether it still skips the same alignment (same target, pin and Lemonade version, under 24 hours), the line printed when it does, and the error context added when the install it then runs fails. | domain | | [`archive_cache.rs`](../engines/lemonade/src/archive_cache.rs) | The cached embeddable-archive layer: SHA-256 verification, verified publication into the cache, and the per-archive lock `ensure_cached_archive` returns as an `ArchiveCacheGuard` to hold while using the bytes. | domain | | [`archive_extract.rs`](../engines/lemonade/src/archive_extract.rs) | The hardened in-process tar.gz and zip extractor for the embeddable archive, and the containment-checked `copy_tree` into the runtime directory. | domain | | [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) | Aligning the llama.cpp backend with the ROCm SDK (Tier 1, Tier 2, revert), including the `backend_versions.json` pin, setting ROCm backend directories aside around a forced reinstall, and skipping an alignment [`alignment_record.rs`](../engines/lemonade/src/alignment_record.rs) recorded as failed. | domain | diff --git a/docs/engine-plugins.md b/docs/engine-plugins.md index ede3f92ecd..d17da92d20 100644 --- a/docs/engine-plugins.md +++ b/docs/engine-plugins.md @@ -79,27 +79,36 @@ without that every later `rocm engines install lemonade`, and every `rocm serve` that installs Lemonade, would download both builds and force-reinstall the pinned backend again. When the revert's install succeeds, the attempt (the active ROCm version, the pins it started from, the llama.cpp builds tried and -the Lemonade version) is recorded in `.rocm-cli-alignment-failed.json` in the -Lemonade runtime tree. For the next 24 hours an install for the same active ROCm -version, pin and Lemonade version does not attempt the alignment: it installs -the backend already pinned without forcing a reinstall, so a backend already -installed is reused, and prints a line saying the alignment could not be done, -when it failed, and how many hours from now an install tries it again (nothing -retries on its own). `rocm engines install lemonade --reinstall` tries it again -at once, because it deletes and re-extracts the runtime tree, record included. A -different active ROCm version, a new Lemonade version (which also re-extracts -the tree), or a record 24 hours old or older (or dated in the future) runs the -alignment as before, and an alignment that succeeds removes the record. The -record is consulted only when an install could align: not with the opt-out set, -on an unsupported platform, with no active ROCm version or an unreadable pin, or -when the pin already equals the active ROCm version, all of which leave it -untouched and say nothing about it. Past those checks, a record that cannot be -read is removed with a warning, before the backend probe decides whether to -align; if it cannot be removed either, the warning says so and names `rocm +the Lemonade version) is recorded in +`.rocm-cli-alignment-failed.json` in the Lemonade runtime tree. For the next 24 +hours an install for the same active ROCm version, pin and Lemonade version +does not attempt the alignment: it installs the backend already +pinned without forcing a reinstall, so a backend already installed is reused, +and prints a line saying the alignment could not be done, when it failed, and +how many hours from now an install tries it again (nothing retries on its own). +If preparing that pinned backend fails (which can happen before anything is +installed, for example when `lemond` does not start), the error says so and that +the alignment was skipped because of the earlier failure, and says that `rocm +engines install lemonade --reinstall` re-extracts the runtime tree without the +record, so it tries the alignment again (downloading the backend again); the +line it adds says only what that command does, not that it fixes the error +beneath. Run at any time, the same command tries the alignment again at once. +A different active ROCm version, a new +Lemonade version (which also re-extracts the tree), or a record 24 hours old or +older (or dated in the future) runs the alignment as before, and an +alignment that succeeds removes the record. The record is consulted only when an +install could align: not with the opt-out set, on an unsupported platform, with +no active ROCm version or an unreadable pin, or when the pin already equals the +active ROCm version, all of which leave it untouched and say nothing about it. +Past those checks, and before the backend probe starts `lemond`, a record that +applies skips the alignment, and one that cannot be read is removed with a +warning; if it cannot be removed either, the warning says so and names `rocm engines install lemonade --reinstall`, which removes it. A record that cannot be written leaves the next install to try again, also with a warning. Setting `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT` neither reads nor changes the -record. +record. The record does not note the rocm-cli version, so upgrading rocm-cli +does not clear it: it still expires after 24 hours, and `--reinstall` clears it +sooner. A pin an earlier alignment kept can go stale: the llama.cpp or TheRock build it names may later be withdrawn upstream. When the pin already equals the active diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 9e95d28e95..ee810e7dbc 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -504,6 +504,14 @@ then `rocm engines install lemonade`. Expected result: that an install run 24 hours from now or later tries it again, and exits 0. - Running the first command again attempts the alignment again (the warning is back), because `--reinstall` removed the record with the runtime tree. +- Not a manual step: when preparing the pinned backend fails on the second + install, its error says so, that the alignment was skipped because of the + earlier failure, and that `rocm engines install lemonade --reinstall` + re-extracts the runtime tree without that record; the record is kept and the + skip line above is not printed. There is no way to make that install fail on + demand, so this is covered by the unit test + `a_skipped_alignment_whose_install_fails_reports_the_error_and_claims_nothing` + in [backend_alignment.rs](../engines/lemonade/src/backend_alignment.rs). To check what a Lemonade install says about its size (no download needed for the refusal): with a debug build, set diff --git a/docs/testing.md b/docs/testing.md index ccd3890548..46cc333dd2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2678,25 +2678,31 @@ cargo test -p rocm-engine-lemonade a_skipped_alignment_whose_install_fails The tests are in [alignment_record.rs](../engines/lemonade/src/alignment_record.rs) and [backend_alignment.rs](../engines/lemonade/src/backend_alignment.rs), one per -state of the record: none, one that skips the alignment (the regression test: -the second install calls no alignment tier and installs without forcing, and its -notice is asserted with the pin file), one for another ROCm version, one 24 -hours old, an unreadable one, one that cannot be written, one that cannot be -removed after a later successful alignment, one removed by a later successful -alignment, one removed by `--reinstall` -(`reinstall_removes_the_record_of_a_failed_alignment` in -[install.rs](../engines/lemonade/src/install.rs)), one the opt-out leaves alone, -and a skipped alignment whose install fails (the error is reported and no notice -is printed). The unreadable record has a row for each place it can be met: every -install that returns before the record is consulted (the opt-out, an unsupported -platform, no active ROCm version or a failed lookup of it, an unreadable pin, -and a pin that already equals the active ROCm version) leaves it as it is and -says nothing; one that gets past those checks removes it with a warning, whether -the alignment then runs, the backend probe fails or the probe selects Vulkan; -and when it cannot be removed, the warning carries the removal's own error and -names `rocm engines install lemonade --reinstall`. Both a record that applies -and an unreadable one are handled before the backend probe starts `lemond`; a -revert whose install fails writes no record; and +state of the record described in the section linked above, and their names say +which: no record (`with_no_record_the_alignment_runs_as_before`), one that +applies (the regression test, +`a_failed_alignment_is_not_repeated_by_the_next_install_within_a_day`: the +second install calls no alignment tier and installs without forcing, and its +notice is asserted with the pin file), one that applies checked before the +backend probe starts `lemond` +(`a_record_that_applies_skips_before_the_backend_probe_runs`), one for another +ROCm version, one a day old, one an early exit never consults (one table test, +a row per exit, with an unreadable record on disk), an unreadable one past +those exits (removed with a warning on every path: probe error, Vulkan and +ROCm, in `an_unreadable_record_is_removed_with_one_warning_on_every_path`; and, +on the ROCm-probe path only, the probe asserting from inside that the removal +and warning came first, in +`an_unreadable_record_is_removed_and_reported_before_the_backend_probe_runs`), one that cannot be removed either, one that cannot be written, one +a later verified alignment removes, one that cannot be removed after that +alignment, a revert whose install fails writing none (asserted in +`align_revert_restores_the_backend_dir_when_the_fallback_install_fails`), the +opt-out leaving one alone, and a skipped alignment whose install fails +(`a_skipped_alignment_whose_install_fails_reports_the_error_and_claims_nothing`: +its error chain asserted in three rows, `lemond` not starting, a transient +failure and the backend retry's own advice, each with the kept record and no +notice). Removal by `--reinstall` is +`reinstall_removes_the_record_of_a_failed_alignment` in +[install.rs](../engines/lemonade/src/install.rs), and `the_record_file_name_matches_the_one_the_e2e_steps_hard_code` reads the e2e steps' copy of the record's file name, so the two cannot drift. The scenario `runtime-lemonade-backend-alignment-failure-remembered` (runtime-33 in diff --git a/engines/lemonade/src/alignment_record.rs b/engines/lemonade/src/alignment_record.rs index 7b787e3b0d..1fa85c3db8 100644 --- a/engines/lemonade/src/alignment_record.rs +++ b/engines/lemonade/src/alignment_record.rs @@ -113,6 +113,11 @@ impl AlignmentFailure { /// Whether the alignment should be skipped now: the same attempt (target, /// pin and Lemonade version) failed less than [`ALIGNMENT_RETRY_AFTER`] ago. /// + /// The record is not keyed on the rocm-cli version: this engine cannot see + /// the per-release version today, so a rocm-cli upgrade does not clear the + /// record. It expires after [`ALIGNMENT_RETRY_AFTER`], and + /// [`RETRY_ALIGNMENT_COMMAND`] clears it sooner. + /// /// A failure time later than `now` (the clock moved back) does not skip: /// trying again costs an install, and a record whose age cannot be known /// must not stop the alignment indefinitely. @@ -154,6 +159,31 @@ impl AlignmentFailure { when = describe_from_now(remaining), ) } + + /// The context added to the error when the record skipped the alignment and + /// preparing the pinned backend then failed. That preparation can fail before + /// anything is installed (no free port, `lemond` not starting or not + /// answering, the backend listing, no supported GPU backend) and Lemonade may + /// pick a backend other than ROCm, so the line claims neither an install nor + /// a backend kind. It says what [`RETRY_ALIGNMENT_COMMAND`] does -- re-extract + /// the runtime tree without the record, so it tries the alignment again and + /// downloads the backend again -- and not that it fixes the error beneath, + /// which may be transient, or that it avoids the pinned backend, which a + /// failing alignment reverts to. + pub(crate) fn skipped_install_failure(&self, now: SystemTime) -> String { + // Called only after `skips_alignment` said yes for this `now`, which needs a + // known age, so the default is never taken. + let age = self.age(now).unwrap_or_default(); + format!( + "preparing Lemonade's llama.cpp backend as pinned for ROCm {pinned} failed; the \ + alignment to ROCm {target} was skipped because it failed {ago}. \ + `{RETRY_ALIGNMENT_COMMAND}` re-extracts the runtime tree without that record, \ + so it tries the alignment again (downloading the backend again)", + pinned = self.pinned_rocm_version, + target = self.target_rocm_version, + ago = describe_ago(age), + ) + } } const HOUR: u64 = 60 * 60; diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index 6976205742..98f5a7018f 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -76,7 +76,10 @@ fn lemonade_backend_alignment_disabled() -> bool { /// ([`crate::alignment_record`]), and for the next /// [`ALIGNMENT_RETRY_AFTER`](crate::alignment_record::ALIGNMENT_RETRY_AFTER) the same /// alignment is not tried again: the backend already pinned is installed without -/// forcing a reinstall, and a line says so and how to try again sooner. +/// forcing a reinstall, and a line says so and how to try again sooner. If that +/// fails, the error says the alignment was skipped because of the record and that +/// `--reinstall` re-extracts the runtime tree without it, so it tries the +/// alignment again. /// /// Returns the aligned version string when one was applied and verified, so the caller /// can surface it in the install response. @@ -229,8 +232,11 @@ fn prepare_llamacpp_backend_for_active_rocm_impl( if failure.skips_alignment(&target_version, &pinned_version, &manifest.version, now) { // Not forced: the backend the revert installed is the one wanted. // The line is printed only once that install has succeeded, since - // it says the pinned backend is in use. - fallback_install(manifest, false)?; + // it says the pinned backend is in use. A failure says the record + // is why no alignment was tried, and how to remove it, whichever + // step of the install failed. + fallback_install(manifest, false) + .with_context(|| failure.skipped_install_failure(now))?; notice(failure.skip_notice(now)); return Ok(None); } @@ -3260,32 +3266,77 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Preparing the skipped alignment's pinned backend fails: the error says so + /// without claiming an install ran or which backend, says why no alignment was + /// tried and what `--reinstall` does, and keeps the error beneath, whether that + /// is `lemond` not starting, a transient failure, or the backend retry's own + /// `--reinstall` advice. That `--reinstall` removes the record is + /// `reinstall_removes_the_record_of_a_failed_alignment` in `install.rs`; that + /// the next install then aligns is `with_no_record_the_alignment_runs_as_before`. #[test] fn a_skipped_alignment_whose_install_fails_reports_the_error_and_claims_nothing() { - let (dir, _) = runtime_dir_with_packaged_pin("record-skip-install-fails"); - let failure = recorded_failure(RECORD_TARGET, FAILED_AT); - write_alignment_failure(&dir, &failure).unwrap(); + const CONTEXT: &str = "preparing Lemonade's llama.cpp backend as pinned for ROCm \ + 7.13.0 failed; the alignment to ROCm 7.99.0 was skipped because it failed 1 hour \ + ago. `rocm engines install lemonade --reinstall` re-extracts the runtime tree \ + without that record, so it tries the alignment again (downloading the backend \ + again)"; + type Install = fn() -> Result<()>; + let rows: [(&str, Install, String); 3] = [ + ( + // The shape `spawn_lemond` returns: nothing was installed, and no + // backend kind is known yet. + "lemond does not start", + || { + Err(std::io::Error::from(std::io::ErrorKind::NotFound)) + .context("failed to start /opt/lemonade/bin/lemond") + }, + format!("{CONTEXT}: failed to start /opt/lemonade/bin/lemond: entity not found"), + ), + ( + "transient", + || bail!("timed out waiting for lemond to start"), + format!("{CONTEXT}: timed out waiting for lemond to start"), + ), + ( + "the backend retry's own advice", + || { + Err(anyhow!("backend download failed")) + .context(crate::llamacpp_backend::BACKEND_INSTALL_RETRY_FAILED) + }, + format!( + "{CONTEXT}: {}: backend download failed", + crate::llamacpp_backend::BACKEND_INSTALL_RETRY_FAILED + ), + ), + ]; + for (label, install, chain) in rows { + let (dir, _) = runtime_dir_with_packaged_pin("record-skip-install-fails"); + let failure = recorded_failure(RECORD_TARGET, FAILED_AT); + write_alignment_failure(&dir, &failure).unwrap(); - let run = prepare_against_record(&dir, false, at(FAILED_AT + HOUR), || { - bail!("backend download failed") - }); + let run = prepare_against_record(&dir, false, at(FAILED_AT + HOUR), install); - assert_eq!( - run.result.unwrap_err().to_string(), - "backend download failed" - ); - assert_eq!(run.align_calls, 0); - assert_eq!(run.installs, [false]); - assert!( - run.notices.is_empty(), - "the notice says the pinned backend is in use, which it is not: {:?}", - run.notices - ); - assert_eq!( - read_alignment_failure(&dir), - StoredAlignmentFailure::Recorded(failure) - ); - fs::remove_dir_all(&dir).ok(); + let error = run.result.unwrap_err(); + assert_eq!( + error.to_string(), + failure.skipped_install_failure(at(FAILED_AT + HOUR)), + "{label}" + ); + assert_eq!(format!("{error:#}"), chain, "{label}"); + assert_eq!(run.align_calls, 0, "{label}"); + assert_eq!(run.installs, [false], "{label}"); + assert!( + run.notices.is_empty(), + "{label}: the notice says the pinned backend is in use, which it is not: {:?}", + run.notices + ); + assert_eq!( + read_alignment_failure(&dir), + StoredAlignmentFailure::Recorded(failure), + "{label}: the record is kept" + ); + fs::remove_dir_all(&dir).ok(); + } } #[test] diff --git a/engines/lemonade/src/llamacpp_backend.rs b/engines/lemonade/src/llamacpp_backend.rs index 3679d5c733..5d5e8f5b11 100644 --- a/engines/lemonade/src/llamacpp_backend.rs +++ b/engines/lemonade/src/llamacpp_backend.rs @@ -87,6 +87,11 @@ pub(crate) fn best_llamacpp_backend_for_host( result } +/// The context [`install_llamacpp_backend_with_retry`] puts on the second +/// failure: the recovery the user is pointed at. +pub(crate) const BACKEND_INSTALL_RETRY_FAILED: &str = "Lemonade backend installation failed \ + again; run `rocm engines install lemonade --reinstall` and then retry `rocm serve`"; + /// Ask Lemonade which llama.cpp backends it supports on this GPU, choose the best /// one (`LLAMACPP_BACKEND_PRIORITY`), install it if necessary, and return its name. /// Retry the backend install itself once, rather than the whole Lemonade runtime @@ -102,10 +107,7 @@ pub(crate) fn install_llamacpp_backend_with_retry( Ok(()) => Ok(()), Err(first_error) => { eprintln!("Lemonade backend installation failed; retrying once: {first_error:#}"); - install().with_context(|| { - "Lemonade backend installation failed again; run `rocm engines install \ - lemonade --reinstall` and then retry `rocm serve`" - }) + install().context(BACKEND_INSTALL_RETRY_FAILED) } } } diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index f798e12479..16f90afbed 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -728,7 +728,8 @@ Feature: Runtime configuration # The first install here is the failure; the second is the one under test. # The failure is remembered for 24 hours; the unit tests in # engines/lemonade/src/backend_alignment.rs cover expiry, another ROCm - # version, an unreadable record and the opt-out. + # version, an unreadable record, a failed install + # after the skip, and the opt-out. # # Technical details: the version the alignment targets is overridden to # 7.99.0 through the `rocm/e2e-test-hooks` variable diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs index ae23ce2dca..6e7813ab3e 100644 --- a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -606,7 +606,11 @@ async fn assert_lemonade_record_cleared(world: &mut E2eWorld) { const UNPUBLISHED_ROCM_VERSION: &str = "7.99.0"; /// The override applies to one command only (`run_rocm_with_scenario_env` -/// consumes it), so each step that runs `rocm` sets it again first. +/// consumes it). So the step that is about to run `rocm` either sets it itself +/// or relies on an immediately preceding step having set it, and a step that +/// runs `rocm` and is followed by another such step re-arms it after its own +/// run: the reinstall Given does, for the shared `When the user installs the +/// Lemonade engine`. fn target_unpublished_rocm_version(world: &mut E2eWorld) { world.command_env.push(( "ROCM_E2E_LEMONADE_ALIGNMENT_TARGET_VERSION",