diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 230544bfa..aee9552b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1020,10 +1020,15 @@ jobs: # serve-33/serve-34/serve-35 (the native-Windows vLLM refusal for `serve` and # `engines install`) are folded in the same way: `@requires-os:windows`, # hermetic, and otherwise run only on the optional self-hosted lane. + # + # runtime-lifecycle-19/20/21 (runtimes installed before and after the ROCm + # 10 layout, side by side) are folded in for the same reason: planted + # runtimes, no GPU or network, and their Windows paths are what this + # lane proves. env: E2E_INCLUDE_LIFECYCLE: "1" E2E_ONLY_LIFECYCLE: "1" - E2E_ALSO_RUN_IDS: "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows" + E2E_ALSO_RUN_IDS: "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows,runtime-lifecycle-side-by-side-across-packaging-change,runtime-lifecycle-side-by-side-uninstall-keeps-the-others,runtime-lifecycle-side-by-side-shared-name-refused" run: | # Derive from CARGO_TARGET_DIR when it is set, falling back to the # default target dir, so a future job-level CARGO_TARGET_DIR cannot diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 41eabb776..e8b24ae67 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -18,9 +18,13 @@ //! also holds `paths_equivalent` and `normalize_path_for_compare`, which adopt //! and uninstall both need, and the on-disk active-runtime marker //! (`ActiveRuntimeMarker`, `active_runtime_marker_path`, -//! `active_runtime_marker_matches`) that activation writes and uninstall reads -//! and clears. The serde shape of [`InstalledRuntimeManifest`] and its probe -//! structs is the on-disk contract of the registry files and of each install's +//! `active_runtime_marker_matches`, `write_active_runtime_marker`, +//! `forget_active_runtime_rollback_target`) that activation and rollback write, +//! and that uninstall reads, removes when it names the removed runtime as the +//! one in use, and otherwise rewrites to drop that runtime as its rollback +//! target; `runtimes_cmd.rs`'s module doc lists every writer and remover. The +//! serde shape of [`InstalledRuntimeManifest`] and its probe structs is the +//! on-disk contract of the registry files and of each install's //! `.rocm-cli-runtime.json`; the marker is user state too, and its serde shape //! is as fixed. @@ -769,6 +773,81 @@ pub(crate) fn active_runtime_marker_path(paths: &AppPaths) -> PathBuf { paths.data_dir.join("runtimes").join("active.json") } +/// Replace the active-runtime marker in one step. +/// +/// Two `rocm` processes can activate a runtime against the same data +/// directory at once (two terminals, a script), so the write goes through the +/// shared atomic writer: each writer stages into its own exclusively created +/// temp file, and the marker is replaced rather than removed first, so a +/// reader always finds a complete marker: on Windows the publish renames first, +/// which replaces in one step. Only when that rename fails, because another +/// process holds the marker open without delete sharing, does it fall back to +/// `ReplaceFileW`, and a `ReplaceFileW` that fails part-way can leave the marker +/// missing, which the writer repairs with a plain rename when that succeeds +/// (see `rocm_core::atomic_write`); two replacements in flight at once through +/// the fallback are not tested. It does not order the two activations: whichever +/// publishes last wins. +pub(crate) fn write_active_runtime_marker( + paths: &AppPaths, + marker: ActiveRuntimeMarker, +) -> Result<()> { + let path = active_runtime_marker_path(paths); + // The shared helper rather than a local temp-file-and-rename: it is the same + // write the failed-activation restore uses, and it is strictly stronger than + // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a + // sharing violation when another process holds the marker open. The helper + // renames first there and falls back to `ReplaceFileW` when that fails. The + // destination is never removed first, on any platform — that would leave a + // window with no marker at all, which runtime resolution and the storage + // retention holds both read as "nothing is active" — and a failed publish + // takes its temp file with it, so a retried activation cannot strand one in + // the folder `rocm runtimes` reads. The one exception is a Windows publish + // that took the old marker away and could not put the new one in place: + // the temp file is then the only copy, so it is kept and the error names + // it; a failed activation removes it again once the restore has put the + // previous marker back (`runtimes_cmd::discard_failed_marker_replacement`). + // It also creates the `runtimes` folder when it is missing. + let bytes = + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; + rocm_core::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write active runtime marker {}", path.display())) +} + +/// Drops `runtime_key` from the active-runtime marker's rollback target, keeping +/// every other field, and returns whether the marker changed. +/// +/// The read-modify-write of the marker for a removed runtime: no marker +/// (nothing, or something other than a file, at its path), one with no rollback +/// target, and one whose rollback target is another runtime are all left alone: +/// it returns `false` and writes nothing. The write is +/// [`write_active_runtime_marker`], the one activation uses. +pub(crate) fn forget_active_runtime_rollback_target( + paths: &AppPaths, + runtime_key: &str, +) -> Result { + let marker_path = active_runtime_marker_path(paths); + // As in `active_runtime_marker_matches`: anything but a file there is no + // marker, so there is no rollback target to drop. + if !marker_path.is_file() { + return Ok(false); + } + let bytes = fs::read(&marker_path) + .with_context(|| format!("failed to read {}", marker_path.display()))?; + let mut marker: ActiveRuntimeMarker = serde_json::from_slice(&bytes) + .with_context(|| format!("failed to parse {}", marker_path.display()))?; + if !marker + .previous_runtime_key + .as_deref() + .is_some_and(|previous| previous.eq_ignore_ascii_case(runtime_key)) + { + return Ok(false); + } + marker.previous_runtime_key = None; + marker.previous_runtime_id = None; + write_active_runtime_marker(paths, marker)?; + Ok(true) +} + #[cfg(test)] mod tests { use super::*; @@ -1774,4 +1853,90 @@ mod tests { ); fs::remove_dir_all(root).ok(); } + + fn marker_with_previous(previous_key: &str) -> ActiveRuntimeMarker { + ActiveRuntimeMarker { + runtime_id: "therock-nightly:gfx1151".to_owned(), + runtime_key: "nightly-pip-gfx1151-7-14-0a1".to_owned(), + manifest_path: PathBuf::from("registry/nightly-pip-gfx1151-7-14-0a1.json"), + install_root: PathBuf::from("wheel/nightly-pip-gfx1151-7-14-0a1"), + previous_runtime_id: Some("therock-release:gfx1151".to_owned()), + previous_runtime_key: Some(previous_key.to_owned()), + activated_at_unix_ms: 1_700_000_000_000, + } + } + + /// A rollback target that names another runtime is not this uninstall's to + /// drop: the marker is left byte for byte as it was. + #[test] + fn forgetting_a_rollback_target_leaves_a_marker_naming_another_runtime_untouched() -> Result<()> + { + let (root, paths) = test_paths("forget-rollback-other-runtime"); + write_active_runtime_marker(&paths, marker_with_previous("release-pip-gfx1151-7-13-0"))?; + let before = fs::read(active_runtime_marker_path(&paths))?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-12-0")?; + + assert!(!changed); + assert_eq!(fs::read(active_runtime_marker_path(&paths))?, before); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A marker with no rollback target has nothing to drop: it is left byte + /// for byte as it was. + #[test] + fn forgetting_a_rollback_target_leaves_a_marker_without_one_untouched() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-none"); + let mut marker = marker_with_previous("release-pip-gfx1151-7-13-0"); + marker.previous_runtime_id = None; + marker.previous_runtime_key = None; + write_active_runtime_marker(&paths, marker)?; + let before = fs::read(active_runtime_marker_path(&paths))?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-13-0")?; + + assert!(!changed); + assert_eq!(fs::read(active_runtime_marker_path(&paths))?, before); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Runtime keys compare without regard to case everywhere else in the + /// registry, so a case variant of the rollback target is the same runtime. + #[test] + fn forgetting_a_rollback_target_matches_its_key_without_regard_to_case() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-case"); + write_active_runtime_marker(&paths, marker_with_previous("release-pip-gfx1151-7-13-0"))?; + + let changed = forget_active_runtime_rollback_target(&paths, "RELEASE-PIP-GFX1151-7-13-0")?; + + assert!(changed); + let after: ActiveRuntimeMarker = + serde_json::from_slice(&fs::read(active_runtime_marker_path(&paths))?)?; + assert_eq!(after.previous_runtime_key, None); + assert_eq!(after.previous_runtime_id, None); + let in_use = "nightly-pip-gfx1151-7-14-0a1"; + assert_eq!(after.runtime_key, in_use); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// With no marker there is no rollback target to drop, and that is not an + /// error: uninstalling a runtime that was never activated must succeed. It + /// leaves the data folder as it found it: no marker, and not even the + /// `runtimes` folder the marker writer would create. + #[test] + fn forgetting_a_rollback_target_without_a_marker_changes_nothing() -> Result<()> { + let (root, paths) = test_paths("forget-rollback-no-marker"); + fs::create_dir_all(&paths.data_dir)?; + + let changed = forget_active_runtime_rollback_target(&paths, "release-pip-gfx1151-7-13-0")?; + + assert!(!changed); + assert!(!active_runtime_marker_path(&paths).exists()); + assert_eq!(fs::read_dir(&paths.data_dir)?.count(), 0); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/runtime_uninstall.rs b/apps/rocm/src/runtime_uninstall.rs index e140cf155..a42f4cb86 100644 --- a/apps/rocm/src/runtime_uninstall.rs +++ b/apps/rocm/src/runtime_uninstall.rs @@ -4,8 +4,9 @@ //! `rocm runtimes uninstall`: planning the removal of a registered runtime, //! re-validating that plan after the confirmation prompt, and applying it -//! (unregistering the runtime, clearing the active and default state that named -//! it, and removing its install root when ROCm CLI owns it). When that root +//! (unregistering the runtime, clearing the active, rollback and default state +//! that named it in both the config and the active-runtime marker, and removing +//! its install root when ROCm CLI owns it). When that root //! holds the Lemonade install its record points at (a self-managed engine //! installs inside the active runtime's folder), the plan says so and applying //! it clears the record, so Lemonade does not keep naming deleted files. @@ -14,10 +15,11 @@ //! `runtimes_cmd.rs` calls into this module from `runtimes()` and re-exports //! `uninstall_runtime`, `should_remove_runtime_install_root`, //! `InstallRootDecision` and `lemonade_record_report_line` for `storage.rs`. -//! The active-runtime marker check this module makes -//! (`active_runtime_marker_matches`, over the `ActiveRuntimeMarker` the -//! activation writes) and the path comparison it needs live in -//! `runtime_registry.rs`, so this module does not depend on `runtimes_cmd.rs`. +//! The active-runtime marker check and update this module makes +//! (`active_runtime_marker_matches` and `forget_active_runtime_rollback_target`, +//! which writes through the same `write_active_runtime_marker` activation uses) +//! and the path comparison it needs live in `runtime_registry.rs`, so this +//! module does not depend on `runtimes_cmd.rs`. use std::fs; use std::path::{Path, PathBuf}; @@ -29,7 +31,8 @@ use rocm_engine_lemonade::RecordCleanup; use crate::cli_report; use crate::runtime_registry::{ active_runtime_marker_matches, active_runtime_marker_path, current_runtime_manifest, - paths_equivalent, runtime_manifest_path, select_runtime_manifest, + forget_active_runtime_rollback_target, paths_equivalent, runtime_manifest_path, + select_runtime_manifest, }; use crate::therock; @@ -360,6 +363,10 @@ pub(crate) fn apply_runtime_uninstall( ) })?; } + } else { + // The marker keeps naming the runtime in use; only a rollback target + // that names the removed runtime is dropped, as the config already did. + forget_active_runtime_rollback_target(paths, &manifest.runtime_key)?; } // Decided from the folder actually removed, not from the plan: the record @@ -505,6 +512,149 @@ mod tests { Ok(()) } + /// Removing the rollback target forgets it in both places that record it. + /// The config already lost it; the active-runtime marker kept naming it as + /// previous, and storage retention reads that field as a runtime to hold. + /// The marker must still name the runtime in use, unchanged. + #[test] + fn uninstalling_the_rollback_target_clears_it_from_the_active_marker() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-rollback-target-marker"); + let previous = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let active = write_test_pip_runtime( + &paths, + "nightly-pip-gfx1151-7-14-0a1", + "therock-nightly:gfx1151", + "7.14.0a1", + 20, + )?; + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &previous.runtime_key)?; + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &active.runtime_key)?; + let read_marker = || -> Result { + Ok(serde_json::from_slice(&fs::read( + active_runtime_marker_path(&paths), + )?)?) + }; + let before = read_marker()?; + assert_eq!( + before.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + + uninstall_runtime(&paths, &mut config, &previous.runtime_key)?; + + let after = read_marker()?; + assert_eq!(after.previous_runtime_key, None, "{after:?}"); + assert_eq!(after.previous_runtime_id, None, "{after:?}"); + assert_eq!(after.runtime_key, before.runtime_key); + assert_eq!(after.runtime_id, before.runtime_id); + assert_eq!(after.manifest_path, before.manifest_path); + assert_eq!(after.install_root, before.install_root); + assert_eq!(after.activated_at_unix_ms, before.activated_at_unix_ms); + assert_eq!(config.previous_runtime_key, None); + assert_eq!( + config.active_runtime_key.as_deref(), + Some(active.runtime_key.as_str()) + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The config and the marker record the rollback target separately and can + /// disagree: here the config has already let it go while the marker still + /// names it. Uninstalling that runtime must clear it from the marker anyway, + /// leaving the runtime in use as it was. + #[test] + fn uninstall_clears_a_rollback_target_only_the_marker_still_names() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-marker-only-rollback-target"); + let previous = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let active = write_test_pip_runtime( + &paths, + "nightly-pip-gfx1151-7-14-0a1", + "therock-nightly:gfx1151", + "7.14.0a1", + 20, + )?; + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &previous.runtime_key)?; + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &active.runtime_key)?; + assert_eq!( + config.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + // The uninstall reads this in-memory config, so nothing is saved. + config.previous_runtime_key = None; + let read_marker = || -> Result { + Ok(serde_json::from_slice(&fs::read( + active_runtime_marker_path(&paths), + )?)?) + }; + let before = read_marker()?; + assert_eq!( + before.previous_runtime_key.as_deref(), + Some(previous.runtime_key.as_str()) + ); + assert_eq!(config.previous_runtime_key, None); + + uninstall_runtime(&paths, &mut config, &previous.runtime_key)?; + + let after = read_marker()?; + assert_eq!(after.previous_runtime_key, None, "{after:?}"); + assert_eq!(after.previous_runtime_id, None, "{after:?}"); + assert_eq!(after.runtime_key, before.runtime_key); + assert_eq!(after.runtime_id, before.runtime_id); + assert_eq!(after.manifest_path, before.manifest_path); + assert_eq!(after.install_root, before.install_root); + assert_eq!(after.activated_at_unix_ms, before.activated_at_unix_ms); + assert_eq!( + config.active_runtime_key.as_deref(), + Some(active.runtime_key.as_str()) + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Something other than a file at the marker path is not a marker: the + /// active-runtime check reads it as no marker, and so must the rollback-target + /// step, or uninstalling an unrelated runtime fails after its folder and + /// registry entry are already gone. + #[test] + fn uninstall_treats_a_folder_at_the_marker_path_as_no_marker() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-marker-path-is-a-folder"); + let runtime = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let marker_path = active_runtime_marker_path(&paths); + fs::create_dir_all(&marker_path)?; + let mut config = RocmCliConfig::default(); + + uninstall_runtime(&paths, &mut config, &runtime.runtime_key)?; + + assert!(!runtime.install_root.exists()); + assert!(!runtime_manifest_path(&paths, &runtime.runtime_key).exists()); + assert!(marker_path.is_dir()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + /// A record that cannot be read may well name the runtime being removed, so /// the uninstall cannot tell whether it is left dangling. When the runtime /// held a Lemonade tree it must say so rather than treat the record as diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 950960a6b..e098ad5a2 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -20,11 +20,18 @@ //! `RuntimeActivationResult` stays //! `pub(crate)`: it is read by `append_update_activate_summary` in //! `update_cmd.rs`'s `apply_runtime_update` flow. The uninstall plan and apply -//! code lives in `runtime_uninstall.rs`, and the active-runtime marker model in -//! `runtime_registry.rs`; `uninstall_runtime`, -//! `should_remove_runtime_install_root`, `InstallRootDecision`, -//! `ActiveRuntimeMarker` and `active_runtime_marker_path` are re-exported from -//! here for `storage.rs` and `main.rs`. +//! code lives in `runtime_uninstall.rs`, and the active-runtime marker model and +//! `write_active_runtime_marker` in `runtime_registry.rs`; activation and +//! rollback here write the marker through it, as uninstall does when it drops a +//! rollback target. A failed activation's restore and three removals touch the +//! marker without it. The restore puts the earlier bytes back with +//! `rocm_core::atomic_write::write_file_atomically`, or removes the marker when +//! there was none before. Uninstalling the runtime in use removes it, and +//! `rocm config set-default-runtime` and `rocm config clear-default-runtime` +//! remove it and ignore a failed removal. +//! `uninstall_runtime`, `should_remove_runtime_install_root`, +//! `InstallRootDecision`, `ActiveRuntimeMarker` and `active_runtime_marker_path` +//! are re-exported from here for `storage.rs` and `main.rs`. //! `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` //! and `setup_runtime_paths` (from `runtime_adopt.rs`) and //! `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported @@ -48,6 +55,7 @@ use crate::runtime_adopt::{ pub(crate) use crate::runtime_adopt::{ newest_installed_runtime_manifest, recover_setup_runtime_registration, setup_runtime_paths, }; +use crate::runtime_registry::write_active_runtime_marker; pub(crate) use crate::runtime_registry::{ ActiveRuntimeMarker, active_runtime_marker_path, current_runtime_manifest, default_runtime_id_matches, normalize_path_for_compare, runtime_keys_text, @@ -875,43 +883,6 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { } } -/// Replace the active-runtime marker in one step. -/// -/// Two `rocm` processes can activate a runtime against the same data -/// directory at once (two terminals, a script), so the write goes through the -/// shared atomic writer: each writer stages into its own exclusively created -/// temp file, and the marker is replaced rather than removed first, so a -/// reader always finds a complete marker: on Windows the publish renames first, -/// which replaces in one step. Only when that rename fails, because another -/// process holds the marker open without delete sharing, does it fall back to -/// `ReplaceFileW`, and a `ReplaceFileW` that fails part-way can leave the marker -/// missing, which the writer repairs with a plain rename when that succeeds -/// (see `rocm_core::atomic_write`); two replacements in flight at once through -/// the fallback are not tested. It does not order the two activations: the last -/// one to publish wins, as before. -fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { - let path = active_runtime_marker_path(paths); - // The shared helper rather than a local temp-file-and-rename: it is the same - // write the failed-activation restore uses, and it is strictly stronger than - // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a - // sharing violation when another process holds the marker open. The helper - // renames first there and falls back to `ReplaceFileW` when that fails. The - // destination is never removed first, on any platform — that would leave a - // window with no marker at all, which runtime resolution and the storage - // retention holds both read as "nothing is active" — and a failed publish - // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. The one exception is a Windows publish - // that took the old marker away and could not put the new one in place: - // the temp file is then the only copy, so it is kept and the error names - // it; a failed activation removes it again once the restore has put the - // previous marker back (`discard_failed_marker_replacement`). It also - // creates the `runtimes` folder when it is missing. - let bytes = - serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; - rocm_core::atomic_write::write_file_atomically(&path, &bytes) - .with_context(|| format!("failed to write active runtime marker {}", path.display())) -} - #[cfg(test)] pub(crate) mod tests { use std::path::{Path, PathBuf}; diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 804e26a09..10ca285ac 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -1491,6 +1491,74 @@ mod tests { } } + /// Pins the production derivation to the literal keys of the side-by-side + /// e2e fixture, from the fixture's inputs. The fixture + /// (tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs) re-derives + /// keys in `Role::key` to plant runtimes without installing them, and its + /// planting step checks those against the same literals in + /// `Role::expected_key`. So a change here fails this test, and a change to + /// the fixture fails the scenario's Given. + #[test] + fn wheel_runtime_key_matches_the_side_by_side_e2e_fixture() { + let composition = |generation: &str, specs: [&str; 4]| WheelRuntimeComposition { + source_layout_generation: generation.to_owned(), + package_specs: specs.iter().map(|spec| (*spec).to_owned()).collect(), + rocm_sdk_target: Some("gfx942".to_owned()), + }; + let cases = [ + ( + TheRockChannel::Release, + "7.14.0", + composition( + "multi-arch-v2", + [ + "rocm[libraries,devel,device-gfx942]==7.14.0", + "torch[device-gfx942]==2.11.0+rocm7.14.0", + "torchvision[device-gfx942]==0.26.0+rocm7.14.0", + "torchaudio==2.11.0+rocm7.14.0", + ], + ), + "release-wheel-multi-arch-7-14-0-992a08553d8be8f0", + ), + ( + TheRockChannel::Release, + "10.0.0", + composition( + "next-v1", + [ + "rocm[libraries,device-gfx942]==10.0.0", + "torch[device-gfx942]==2.13.0+rocm10.0.0", + "torchvision[device-gfx942]==0.28.0+rocm10.0.0", + "torchaudio==2.11.0.2+rocm10.0.0", + ], + ), + "release-wheel-multi-arch-10-0-0-3d0e256f664a960d", + ), + ( + TheRockChannel::Nightly, + "10.1.0a20260822", + composition( + "multi-arch-v2", + [ + "rocm[libraries,device-gfx942]==10.1.0a20260822", + "torch[device-gfx942]==2.12.0+rocm10.1.0a20260822", + "torchvision[device-gfx942]==0.27.0+rocm10.1.0a20260822", + "torchaudio==2.12.0+rocm10.1.0a20260822", + ], + ), + "nightly-wheel-multi-arch-10-1-0a20260822-95e3a823499c530b", + ), + ]; + for (channel, version, composition, expected) in cases { + assert_eq!( + wheel_runtime_key(channel, version, &composition), + expected, + "the production key derivation changed; update these literals, and `Role::key` \ + and `Role::expected_key` in runtime_side_by_side_steps.rs, together" + ); + } + } + #[test] fn aggregate_wheel_runtime_key_separates_device_payloads_at_one_version() { let base = WheelRuntimeComposition { diff --git a/docs/architecture.md b/docs/architecture.md index 9275d8fbd..fab7a48a3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -105,10 +105,10 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) | Registering runtimes `rocm install sdk` did not install (import, adopt, recovering the `rocm setup` runtime) and the one ready-runtime rule (`select_single_ready_runtime`). | domain | | [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) | Settling an engine onto a runtime after an SDK install (`finish_sdk_install`, `settle_engine_install`): engine-dependency and device checks and the post-install verdict. | relocation | | [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) | Whether an engine install realigns the runtime's torch to the engine's pin or keeps it as an intended divergence, running and reporting that, and the manifest readers it uses. | domain | -| [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) | The installed-runtime registry: the manifest model, where manifests live, writing and loading them, selection and activation validation, and the active-runtime marker. | domain | +| [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) | The installed-runtime registry: the manifest model, where manifests live, writing and loading them, selection and activation validation, and the active-runtime marker with the atomic writer activation, rollback and uninstall use (a failed activation's restore writes or removes it directly, and `rocm config set-default-runtime`/`clear-default-runtime` remove it). | domain | | [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) | Reconciling running local servers with a runtime being activated or rolled back: classifying each, the report, and moving stale ones with `--restart-services`. | domain | -| [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) | `rocm runtimes uninstall`: planning, revalidating after the prompt, and applying the removal of a registered runtime, including clearing a Lemonade record inside its folder. | domain | -| [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) | `rocm runtimes` (`runtimes`): activation and rollback with the active-runtime marker write, and the runtime list. | relocation | +| [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) | `rocm runtimes uninstall`: planning, revalidating after the prompt, and applying the removal of a registered runtime, including dropping it as the active or rollback runtime in the config and the active-runtime marker and clearing a Lemonade record inside its folder. | domain | +| [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) | `rocm runtimes` (`runtimes`): activation and rollback, which write the active-runtime marker through `runtime_registry.rs`, and the runtime list. | relocation | | [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) | `rocm serve` (`serve`): engine selection (`select_serve_engine`), the AIM pre-flight, and the GPU and device notes folded into the summary. | relocation | | [`serve_port.rs`](../apps/rocm/src/serve_port.rs) | The `rocm serve` port policy: turns an automatic or explicit `--port` request into one leased port before any manifest, log, key or child exists. | domain | | [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) | The pure transforms `serve()` applies to an engine's launch recipe hint: the catalog mapping, then `--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and `--engine-env`. | relocation | @@ -861,6 +861,7 @@ Gherkin `.feature` files under `features/` backed by Rust step functions that dr | [`tests/e2e/planner_steps.rs`](../tests/e2e-cucumber/tests/e2e/planner_steps.rs) | Steps for the planning-assistant scenarios in [`chat.feature`](../tests/e2e-cucumber/features/chat.feature), with a scripted mock proving a request is handed over once and its plan is the one shown. | flat | | [`tests/e2e/remote_steps.rs`](../tests/e2e-cucumber/tests/e2e/remote_steps.rs) | Steps for [`remote.feature`](../tests/e2e-cucumber/features/remote.feature): `rocm remote` against the `fake-tailscale` stand-in and a status document the step writes. | flat | | [`tests/e2e/runtime_lifecycle_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs) | Steps for [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` activate, rollback, uninstall and import over planted read-only tarball runtimes. | flat | +| [`tests/e2e/runtime_side_by_side_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs) | Steps for the side-by-side scenarios in [`runtime_lifecycle.feature`](../tests/e2e-cucumber/features/runtime_lifecycle.feature): `rocm runtimes` list, activate and uninstall over managed wheel runtimes planted as installs from before and after the ROCm 10 layout leave them, comparing their install records byte for byte. | flat | | [`tests/e2e/runtime_steps.rs`](../tests/e2e-cucumber/tests/e2e/runtime_steps.rs) | Steps for [`runtime_setup.feature`](../tests/e2e-cucumber/features/runtime_setup.feature): SDK install previews and installs, runtime folders reached through a link, and the managed runtime they leave active. | flat | | [`tests/e2e/service_cleanup_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs) | Steps for [`service_record_cleanup.feature`](../tests/e2e-cucumber/features/service_record_cleanup.feature): `rocm services remove` and `prune` over planted service records, asserting on the files that survive, and [`service_record_unqueryable.feature`](../tests/e2e-cucumber/features/service_record_unqueryable.feature): a record whose process cannot be queried (the `e2e-test-hooks` build reads `ROCM_E2E_UNQUERYABLE_PIDS`) is stopped as unverified, refused by `remove`, and cleared by `remove --force` only when no recorded process is confirmed running. | flat | | [`tests/e2e/service_stop_steps.rs`](../tests/e2e-cucumber/tests/e2e/service_stop_steps.rs) | Steps for [`service_stop.feature`](../tests/e2e-cucumber/features/service_stop.feature) and [`engine_ipc.feature`](../tests/e2e-cucumber/features/engine_ipc.feature): `rocm services stop` and the daemon's `stop_server` tool against a planted service record, and how the daemon reads what an engine child prints, with a stand-in `rocm` staged beside a copy of `rocmd`; and the confirmed stop against a real recorded process, a recorded PID that now belongs to a different process, and a parent that has started a child. | flat | diff --git a/docs/testing.md b/docs/testing.md index a21272ebc..7028e25bf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1844,6 +1844,17 @@ and `config set-engine --runtime-id` store a selector without resolving it. For `runtimes activate`, `@id:runtime-lifecycle-case-twin-selector-refused` also checks that nothing became active and that naming one key exactly then works. +Runtimes installed before and after the ROCm 10 layout stay usable side by +side: `@id:runtime-lifecycle-side-by-side-across-packaging-change` lists and +activates a ROCm 7 runtime, a ROCm 10 release runtime and a ROCm 10 nightly +runtime without rewriting their install records, +`@id:runtime-lifecycle-side-by-side-uninstall-keeps-the-others` uninstalls one +and checks the others keep their folders and records, and +`@id:runtime-lifecycle-side-by-side-shared-name-refused` checks that the runtime +id the two release runtimes share is refused with both keys named. +Uninstalling a runtime that is the rollback target clears it as the rollback +target in both the config and the active-runtime marker. + Developer-only previous-runtime regression check: ```bash @@ -3638,25 +3649,27 @@ needs a non-root runner. `@id:lifecycle-linux-uninstall-keep-binaries-keeps-path The `windows-release` job of the required `windows-build-and-test` lane additionally folds in the hermetic `comfyui-01`/`comfyui-02`/`comfyui-03` -scenarios and the native-Windows vLLM refusal scenarios -`serve-33`/`serve-34`/`serve-35` (none of them `@lifecycle`) via `E2E_ALSO_RUN_IDS`, a -comma-separated list of `@id`s: +scenarios, the native-Windows vLLM refusal scenarios +`serve-33`/`serve-34`/`serve-35`, and the side-by-side runtime scenarios +`runtime-lifecycle-19`/`runtime-lifecycle-20`/`runtime-lifecycle-21` (none of +them `@lifecycle`) via `E2E_ALSO_RUN_IDS`, a comma-separated list of `@id`s: ```powershell $env:E2E_INCLUDE_LIFECYCLE = "1" $env:E2E_ONLY_LIFECYCLE = "1" -$env:E2E_ALSO_RUN_IDS = "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows" +$env:E2E_ALSO_RUN_IDS = "comfyui-uv-install-failure-names-log,comfyui-uv-install-progress-streamed,comfyui-ambiguous-runtime-actionable,serve-vllm-refused-on-native-windows,serve-vllm-install-refused-on-native-windows,serve-vllm-install-with-runtime-refused-on-native-windows,runtime-lifecycle-side-by-side-across-packaging-change,runtime-lifecycle-side-by-side-uninstall-keeps-the-others,runtime-lifecycle-side-by-side-shared-name-refused" cargo xtask e2e ``` -These three were made hermetic and Windows-portable without being tagged -`@lifecycle` — they're ComfyUI install scenarios, not release packaging/ -install/uninstall ones, and tagging them `@lifecycle` would drop them out of -the full-suite runs in `nightly.yml` and `e2e-selfhosted.yml`, neither of -which sets `E2E_INCLUDE_LIFECYCLE`. `E2E_ALSO_RUN_IDS` selects them into this -lane instead, so this is the one required lane that guarantees Windows -coverage for them; the self-hosted Windows lane that otherwise runs them is -optional. +All nine are hermetic (planted fixture binaries or runtimes, no GPU, no +network) and Windows-portable without being tagged `@lifecycle` — they are +ComfyUI install, vLLM refusal and runtime-selection scenarios, not release +packaging/install/uninstall ones, and tagging them `@lifecycle` would drop +them out of the full-suite runs in `nightly.yml` and `e2e-selfhosted.yml`, +neither of which sets `E2E_INCLUDE_LIFECYCLE`. `E2E_ALSO_RUN_IDS` selects them +into this lane instead, so this is the one required lane that guarantees +Windows coverage for them; the self-hosted Windows lane that otherwise runs +them is optional. Heavy CI runs the matching set per platform. The scenarios reject untrusted signers, bad checksums, bad detached signatures, and missing required signature diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index d005021bc..6ccf280c5 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -148,7 +148,7 @@ Scenarios carry stable-id and capability tags: | `@requires-oom-fault-injection` | Needs the `rocm` binary under test to carry the test-only `e2e-oom-fault-injection` hook, which fabricates a managed launch that runs out of GPU memory so the OOM guidance can be verified without a GPU. Unlike every other capability this one **cannot be probed** — no product command exposes it — so `xtask e2e` reports it via `ROCM_E2E_OOM_FAULT_INJECTION=1`, set only when xtask compiled the binary itself with the feature. A prebuilt `ROCM_CLI_BINARY` (the self-hosted lanes' shipping release build) has the hook compiled out, so these scenarios resolve to **skip** there and run on the mock lane. | | `@serve-timeout:` | Lengthen the serve-readiness wait for a genuinely slow serve (e.g. a large model). | | `@nightly` | Expensive scenario skipped by default; included when `E2E_INCLUDE_NIGHTLY=1`. | -| `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios and the native-Windows vLLM refusal scenarios serve-33/34/35. | +| `@lifecycle` | Expensive, OS-mutating release-lifecycle scenario (packaging + real installer + install/uninstall). Skipped by default; included when `E2E_INCLUDE_LIFECYCLE=1`. `E2E_ONLY_LIFECYCLE=1` selects only this set without bypassing expectation resolution. A caller can additionally fold specific non-`@lifecycle` scenarios into that restricted set via `E2E_ALSO_RUN_IDS` (comma-separated `@id`s), without tagging them `@lifecycle` — which would also pull them out of the full-suite lanes that don't set `E2E_INCLUDE_LIFECYCLE`. `ci.yml`'s Windows lane uses this to fold in the hermetic comfyui-01/02/03 scenarios, the native-Windows vLLM refusal scenarios serve-33/34/35, and the side-by-side runtime scenarios runtime-lifecycle-19/20/21. | Known bugs are **not** tagged in the `.feature` files — they live in `expectations.toml`, keyed by `@id`, each with a `when = { ... }` condition (e.g. diff --git a/tests/e2e-cucumber/features/runtime_lifecycle.feature b/tests/e2e-cucumber/features/runtime_lifecycle.feature index 5f8da06c4..3cd8681b4 100644 --- a/tests/e2e-cucumber/features/runtime_lifecycle.feature +++ b/tests/e2e-cucumber/features/runtime_lifecycle.feature @@ -262,3 +262,49 @@ Feature: Runtime lifecycle state machine And the server is still counted as left on the first runtime And the switch to the second runtime was kept And the server record still names the first runtime + + # ROCm 10 is published in its own layout, beside the canonical one the CLI + # installed from before. A runtime installed from it records that layout, and + # its key carries a fingerprint of the exact packages it was built from; a + # runtime installed before that layout existed has no top-level + # `source_layout_generation` or `devel` field: it records its layout and + # toolchain only inside `wheel_composition`. Both kinds stay installed side by + # side, so the CLI has to keep reading the older record as it was written while + # it lists, switches between and removes the newer ones. These scenarios plant + # three managed runtimes as an install would leave them — a registry entry, the + # same record inside the runtime's own folder, and stub files where the + # readiness check looks — without downloading anything. Every command runs + # with the startup update check disabled, since it would otherwise ask the + # package index about the planted runtimes. The Windows release lane in CI + # selects them by id, so they run on both supported platforms. + @id:runtime-lifecycle-side-by-side-across-packaging-change + Scenario: runtime-lifecycle-19 - Runtimes installed before and after the ROCm 10 layout are used side by side + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + When the user lists the installed runtimes + Then all three runtimes are listed as ready, each under its own key + When the user activates each of the three runtimes in turn + Then each one becomes the runtime in use in turn + And none of the three runtimes' install records was rewritten + + @id:runtime-lifecycle-side-by-side-uninstall-keeps-the-others + Scenario: runtime-lifecycle-20 - Uninstalling one side-by-side runtime leaves the others usable + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + And the nightly runtime is in use after the ROCm 10 release runtime + When the user uninstalls the ROCm 10 release runtime + Then that runtime's folder and record are gone + And the other two runtimes keep their folders and unchanged records + And the nightly runtime is still in use, with no rollback target left + When the user uninstalls the runtime in use + Then no runtime is in use + When the user activates the ROCm 7 runtime + Then the ROCm 7 runtime is the runtime in use + + # The ROCm 7 and ROCm 10 release runtimes share a name: the runtime id is + # derived from the channel and GPU family alone. Picking either silently could + # put the user on the wrong ROCm major version. + @id:runtime-lifecycle-side-by-side-shared-name-refused + Scenario: runtime-lifecycle-21 - A name shared by side-by-side runtimes is refused with each runtime named + Given a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family + When the user activates a runtime by the name the two release runtimes share + Then the CLI refuses and names both release runtimes + And no runtime is in use diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 6e8db6c57..8e92631e9 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -39,6 +39,7 @@ mod e2e { pub mod planner_steps; pub mod remote_steps; pub mod runtime_lifecycle_steps; + pub mod runtime_side_by_side_steps; pub mod runtime_steps; pub mod service_cleanup_steps; pub mod service_stop_steps; @@ -195,6 +196,11 @@ pub struct E2eWorld { /// dir, captured logs). `Some` only for `@lifecycle` scenarios; all its paths /// are rooted in `isolated_root` so teardown removes them with the temp dir. pub lifecycle: Option, + /// The runtimes a side-by-side scenario planted, with the bytes of their + /// install records as planted and what each activation left behind, so a + /// later step can tell an untouched record from a rewritten one. `Some` + /// only for the side-by-side scenarios in `runtime_lifecycle.feature`. + pub side_by_side: Option, /// Raw text of `skills/rocm-doctor/reference.md`, loaded by the rocm-doctor /// skill scenarios. That document is the EXPECTED-value fixture for the /// skill↔CLI contract checks — see `e2e::skill_steps`. @@ -400,6 +406,7 @@ impl Default for E2eWorld { comfyui_baseline_torch_stack: None, comfyui_baseline_distributions: None, lifecycle: None, + side_by_side: None, skill_reference: None, skill_symptom: None, symptom_paste: None, diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index c0bca4e7a..24574e929 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -1067,7 +1067,7 @@ async fn used_comfyui_install(world: &mut E2eWorld) { /// `tests/e2e/tui_driver.rs` (through `spawn_binary_with_env`, which overlays /// `command_env` onto the child and leaves it in place). It does not reach /// plain `run_rocm` or `run_rocm_ok`, which ignore `command_env`. -fn disable_startup_update_check(world: &mut E2eWorld) { +pub(crate) fn disable_startup_update_check(world: &mut E2eWorld) { world .command_env .push(("ROCM_CLI_DISABLE_STARTUP_UPDATE_CHECK", "1".into())); diff --git a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs index 1838780c1..0ccfac83b 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs @@ -82,8 +82,9 @@ fn plant_runtime(world: &E2eWorld, key: &str, family: &str) -> PathBuf { /// The runtime registry under the scenario's isolated data dir — where the CLI /// reads `.json` manifests from. One definition, so a layout change cannot -/// leave the planting and the counting looking in different places. -fn registry_dir(world: &E2eWorld) -> PathBuf { +/// leave the planting and the counting looking in different places; the +/// side-by-side steps read it from here too. +pub(crate) fn registry_dir(world: &E2eWorld) -> PathBuf { world .isolated_root .as_ref() @@ -222,6 +223,38 @@ fn ids_listed_under<'a>(report: &'a str, section: &str) -> Vec<&'a str> { .collect() } +/// The isolated `config.json`, where the CLI records the active runtime. +pub(crate) fn config_path(world: &E2eWorld) -> PathBuf { + world.app_paths().config_dir.join("config.json") +} + +/// The isolated active-runtime marker (`/runtimes/active.json`) — what +/// the next serve would actually use. +pub(crate) fn active_marker_path(world: &E2eWorld) -> PathBuf { + world + .app_paths() + .data_dir + .join("runtimes") + .join("active.json") +} + +/// A JSON file the CLI writes, parsed, or `None` when there is no such file — +/// which is how the CLI leaves the config and the marker before anything was +/// activated, and the marker after the runtime in use was uninstalled. Any +/// other read failure, and a file that is not JSON, fail the step. +pub(crate) fn read_json_if_present(path: &Path) -> Option { + let text = match std::fs::read_to_string(path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => panic!("failed to read {}: {error}", path.display()), + }; + Some( + serde_json::from_str(&text).unwrap_or_else(|error| { + panic!("{} is not valid JSON: {error}\n{text}", path.display()) + }), + ) +} + /// The runtime key recorded in the isolated `config.json`. /// /// The marker (see [`active_runtime_key`]) cannot be used by the scenario that @@ -229,45 +262,27 @@ fn ids_listed_under<'a>(report: &'a str, section: &str) -> Vec<&'a str> { /// anyway: it is written first, so it is the half that would be left naming the /// new runtime if the restore did not run. fn config_active_runtime_key(world: &E2eWorld) -> Option { - let config_dir = world - .isolate_env() - .into_iter() - .find(|(key, _)| *key == "ROCM_CLI_CONFIG_DIR") - .map(|(_, value)| PathBuf::from(value)) - .expect("isolate_env did not set ROCM_CLI_CONFIG_DIR"); - let config = config_dir.join("config.json"); - let text = std::fs::read_to_string(&config) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", config.display())); - let value: serde_json::Value = serde_json::from_str(&text) - .unwrap_or_else(|error| panic!("{} is not valid JSON: {error}", config.display())); - value + let config = config_path(world); + read_json_if_present(&config) + .unwrap_or_else(|| panic!("failed to read {}: it does not exist", config.display())) .get("active_runtime_key") .and_then(serde_json::Value::as_str) .map(str::to_owned) } -/// The runtime key recorded in the isolated active-runtime marker -/// (`/runtimes/active.json`) — what the next serve would actually use. +/// The runtime key recorded in the isolated active-runtime marker. /// /// Parsed as JSON rather than substring-matched on the file: once a second /// runtime has been activated the marker also carries `previous_runtime_key`, /// so a `contains` check would be satisfied by either key and could not tell a /// refused switch from a completed one. fn active_runtime_key(world: &E2eWorld) -> String { - let root = world.isolated_root.as_ref().expect("no isolated root"); - let marker = root - .path() - .join("data") - .join("runtimes") - .join("active.json"); - let text = std::fs::read_to_string(&marker) - .unwrap_or_else(|error| panic!("failed to read {}: {error}", marker.display())); - let value: serde_json::Value = serde_json::from_str(&text) - .unwrap_or_else(|error| panic!("{} is not valid JSON: {error}", marker.display())); - value + let marker = active_marker_path(world); + read_json_if_present(&marker) + .unwrap_or_else(|| panic!("failed to read {}: it does not exist", marker.display())) .get("runtime_key") .and_then(serde_json::Value::as_str) - .unwrap_or_else(|| panic!("no runtime_key in {}:\n{text}", marker.display())) + .unwrap_or_else(|| panic!("no runtime_key in {}", marker.display())) .to_owned() } @@ -397,12 +412,7 @@ async fn two_runtimes_and_service_without_runtime(world: &mut E2eWorld) { /// written normally. #[given("the active runtime marker cannot be written")] async fn marker_cannot_be_written(world: &mut E2eWorld) { - let root = world.isolated_root.as_ref().expect("no isolated root"); - let marker = root - .path() - .join("data") - .join("runtimes") - .join("active.json"); + let marker = active_marker_path(world); std::fs::remove_file(&marker) .unwrap_or_else(|error| panic!("failed to remove {}: {error}", marker.display())); std::fs::create_dir_all(&marker) diff --git a/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs new file mode 100644 index 000000000..00ab793e3 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/runtime_side_by_side_steps.rs @@ -0,0 +1,800 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for the side-by-side scenarios in `runtime_lifecycle.feature`: three +//! managed wheel runtimes for one GPU family — a ROCm 7 runtime installed before +//! the ROCm 10 layout existed, a ROCm 10 release runtime from that layout, and a +//! ROCm 10 nightly prerelease — listed, activated, uninstalled and selected by +//! the name two of them share. +//! +//! The runtimes are planted, not installed: a real install downloads several +//! GiB of wheels. Each record holds what the wheel install in `therock.rs` +//! writes for its role (see [`Role`]): a registry entry and the same manifest in +//! the folder's own `.rocm-cli-runtime.json`, with every field the writing +//! version serialized. The readiness check (`runtime_usability_status` in +//! `apps/rocm/src/runtime_registry.rs`) reads none of the files it checks and +//! runs nothing: it asks for the install folder and the probe's SDK root and bin +//! to be folders, and for the folder's local manifest, the Python executable and +//! a listed `amdhip64` and `hipblas` library to be files, so stub files stand in +//! for all of them. +//! +//! The local manifest is a full copy rather than the `{}` some other fixtures +//! write, because uninstall parses it before deciding the folder's fate +//! (`local_runtime_manifest_matches` in `apps/rocm/src/runtime_uninstall.rs`): +//! one that does not parse as a manifest fails the uninstall with a parse +//! error, and only one naming the same key, id and folder as the registry entry +//! lets the folder be removed. +//! +//! Every command runs with the startup update check disabled. With a runtime +//! installed, a command otherwise first asks the package index whether that +//! runtime has an update, which is network traffic and cache writes these +//! scenarios are not about. + +use std::path::{Path, PathBuf}; + +use cucumber::{given, then, when}; +use e2e_cucumber::cli_failure_report; +use e2e_cucumber::fixture_binary::fixture_target_name; +use sha2::{Digest as _, Sha256}; + +use super::comfyui_steps::disable_startup_update_check; +use super::runtime_lifecycle_steps::{ + active_marker_path, config_path, read_json_if_present, registry_dir, +}; +use crate::E2eWorld; + +/// Shared by the ROCm 7 and the ROCm 10 release runtimes: `rocm install sdk` +/// derives the id from the channel and GPU family alone, so both carry it, and +/// only their keys tell them apart. +const SHARED_RELEASE_RUNTIME_ID: &str = "therock-release:gfx94X-dcgpu"; +const NIGHTLY_RUNTIME_ID: &str = "therock-nightly:gfx94X-dcgpu"; +/// The family `rocm install sdk` resolves for a host reporting `gfx942`. +const FAMILY: &str = "gfx94X-dcgpu"; +const DEVICE_TARGET: &str = "gfx942"; + +/// One of the three planted runtimes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Role { + /// ROCm 7.14.0 from the canonical release stream, installed by the CLI as + /// it stood just before the ROCm 10 layout was added (commit `a6fa76db`'s + /// parent). That install already recorded a wheel composition and keyed the + /// runtime by its fingerprint, but wrote neither `source_layout_generation` + /// nor `devel`, which did not exist yet, and always installed the compiler + /// toolchain. + Rocm7, + /// ROCm 10.0.0, a stable release and so on the release channel only, from + /// the ROCm 10 (`next`) layout, installed by the current CLI without the + /// toolchain (now opt-in). + Rocm10Release, + /// The ROCm 10.1 nightly prerelease `10.1.0a20260822`, the last nightly the + /// canonical layout published: nightlies from 2026-08-23 on come from the + /// `next` layout instead. So this one records the canonical layout + /// (`multi-arch-v2`), installed by the current CLI without the toolchain. + Rocm10Nightly, +} + +impl Role { + const ALL: [Self; 3] = [Self::Rocm7, Self::Rocm10Release, Self::Rocm10Nightly]; + + const fn channel(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Release => "release", + Self::Rocm10Nightly => "nightly", + } + } + + const fn runtime_id(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Release => SHARED_RELEASE_RUNTIME_ID, + Self::Rocm10Nightly => NIGHTLY_RUNTIME_ID, + } + } + + const fn version(self) -> &'static str { + match self { + Self::Rocm7 => "7.14.0", + Self::Rocm10Release => "10.0.0", + Self::Rocm10Nightly => "10.1.0a20260822", + } + } + + /// The layout generation the install recorded in its composition. + const fn layout_generation(self) -> &'static str { + match self { + Self::Rocm7 | Self::Rocm10Nightly => "multi-arch-v2", + Self::Rocm10Release => "next-v1", + } + } + + /// The pip index the install read from, which it records both as + /// `index_url` and as `selected_artifact_url`. The hosts stand in for the + /// published ones the way `ROCM_CLI_THEROCK_*_PIP_BASE` overrides do; the + /// install records whichever base it resolved. + const fn index_url(self) -> &'static str { + match self { + Self::Rocm7 => "https://example.invalid/rocm/whl-multi-arch", + Self::Rocm10Release => "https://example.invalid/rocm/whl-next", + Self::Rocm10Nightly => "https://example.invalid/nightlies/whl-multi-arch", + } + } + + /// Whether the install included the compiler toolchain: always before it + /// became opt-in, and not by default since. + const fn includes_devel(self) -> bool { + matches!(self, Self::Rocm7) + } + + /// `torch`, `torchvision` and `torchaudio` as the install pinned them. + const fn torch_stack(self) -> [&'static str; 3] { + match self { + Self::Rocm7 => [ + "2.11.0+rocm7.14.0", + "0.26.0+rocm7.14.0", + "2.11.0+rocm7.14.0", + ], + Self::Rocm10Release => [ + "2.13.0+rocm10.0.0", + "0.28.0+rocm10.0.0", + "2.11.0.2+rocm10.0.0", + ], + Self::Rocm10Nightly => [ + "2.12.0+rocm10.1.0a20260822", + "0.27.0+rocm10.1.0a20260822", + "2.12.0+rocm10.1.0a20260822", + ], + } + } + + /// The exact specs handed to `uv`, as `therock_pip_package_specs` builds + /// them. + fn package_specs(self) -> Vec { + let device = format!("device-{DEVICE_TARGET}"); + let extras = if self.includes_devel() { + "libraries,devel" + } else { + "libraries" + }; + let [torch, torchvision, torchaudio] = self.torch_stack(); + vec![ + format!("rocm[{extras},{device}]=={}", self.version()), + format!("torch[{device}]=={torch}"), + format!("torchvision[{device}]=={torchvision}"), + format!("torchaudio=={torchaudio}"), + ] + } + + /// The key `wheel_runtime_key` in `apps/rocm/src/therock.rs` derives: the + /// channel and version, then the first 16 hex digits of a SHA-256 over the + /// composition. The derivation is the same before and after the ROCm 10 + /// layout, so the layout generation inside the composition is what keeps a + /// ROCm 10 key from ever colliding with a canonical one. Planting checks the + /// result against [`Self::expected_key`], the literal keys the unit test + /// `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in `therock.rs` + /// derives with the production code from the same inputs. + fn key(self) -> String { + let mut hasher = Sha256::new(); + hasher.update(self.layout_generation().as_bytes()); + for spec in self.package_specs() { + hasher.update([0]); + hasher.update(spec.as_bytes()); + } + hasher.update([0]); + hasher.update(b"rocm-sdk-target"); + hasher.update([0]); + hasher.update(DEVICE_TARGET.as_bytes()); + let fingerprint = format!("{:x}", hasher.finalize()); + format!( + "{}-wheel-multi-arch-{}-{}", + self.channel(), + self.version(), + &fingerprint[..16] + ) + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '-' + } + }) + .collect() + } + + /// The key [`Self::key`] must produce: the literals + /// `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in `therock.rs` + /// gets from the production `wheel_runtime_key` for the same inputs. The + /// unit test ties these literals to the production derivation, and + /// planting ties them to this fixture, so neither side can drift alone. + const fn expected_key(self) -> &'static str { + match self { + Self::Rocm7 => "release-wheel-multi-arch-7-14-0-992a08553d8be8f0", + Self::Rocm10Release => "release-wheel-multi-arch-10-0-0-3d0e256f664a960d", + Self::Rocm10Nightly => "nightly-wheel-multi-arch-10-1-0a20260822-95e3a823499c530b", + } + } + + /// What `rocm runtimes list` must report for the compiler toolchain. + const fn toolchain(self) -> &'static str { + if self.includes_devel() { + "included" + } else { + "excluded" + } + } +} + +/// A planted runtime and the bytes of its two install records as planted. +#[derive(Debug)] +struct PlantedRuntime { + role: Role, + key: String, + install_root: PathBuf, + registry_path: PathBuf, + registry_bytes: Vec, + local_manifest_bytes: Vec, +} + +impl PlantedRuntime { + fn local_manifest_path(&self) -> PathBuf { + self.install_root.join(".rocm-cli-runtime.json") + } +} + +/// What one `rocm runtimes activate` left behind. +#[derive(Debug)] +struct Activation { + role: Role, + in_use: InUse, +} + +/// What the CLI's two records of the runtime in use name: the config +/// (`config.json`) and the active-runtime marker (`/runtimes/active.json`). +/// `None` throughout means the record names nothing, or does not exist. +#[derive(Debug, PartialEq, Eq)] +struct InUse { + config_active: Option, + config_previous: Option, + config_default_id: Option, + marker_key: Option, + marker_id: Option, + marker_previous_key: Option, + marker_previous_id: Option, +} + +/// Per-scenario state, held on the World. +#[derive(Debug)] +pub struct SideBySide { + runtimes: Vec, + activations: Vec, +} + +impl SideBySide { + fn runtime(&self, role: Role) -> &PlantedRuntime { + self.runtimes + .iter() + .find(|runtime| runtime.role == role) + .unwrap_or_else(|| panic!("{role:?} runtime was not planted")) + } + + fn key(&self, role: Role) -> &str { + &self.runtime(role).key + } +} + +const fn state(world: &E2eWorld) -> &SideBySide { + world + .side_by_side + .as_ref() + .expect("no side-by-side runtimes were planted in this scenario") +} + +fn key(world: &E2eWorld, role: Role) -> String { + state(world).key(role).to_owned() +} + +fn write_file(path: &Path, contents: &[u8]) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", parent.display())); + } + std::fs::write(path, contents) + .unwrap_or_else(|error| panic!("failed to write {}: {error}", path.display())); +} + +/// The `site-packages` of a venv at `install_root`, as Python reports it. +fn site_packages(install_root: &Path) -> PathBuf { + if cfg!(windows) { + install_root.join("Lib").join("site-packages") + } else { + install_root + .join("lib") + .join("python3.12") + .join("site-packages") + } +} + +/// Where `rocm_sdk.find_libraries` finds `shortname` inside a package root. +fn library_path(package_root: &Path, shortname: &str) -> PathBuf { + if cfg!(windows) { + package_root.join("bin").join(format!("{shortname}.dll")) + } else { + package_root.join("lib").join(format!("lib{shortname}.so")) + } +} + +/// The `rocm_sdk` probe `rocm install sdk` records, in the shape the probe +/// script in `apps/rocm/src/therock/probe.rs` prints: every package root's `bin` +/// and `lib` folders, the venv's scripts folder, and an SDK root that is the +/// `devel` package when the toolchain is installed and the first package root +/// otherwise, with a CMake path only for `devel`. The script also lists the +/// installed `rocm*` distributions under `packages`; that list is left empty +/// here because nothing on these paths (listing, activating, uninstalling) +/// reads it. Two libraries stand for the full list, being the two the +/// readiness check asks for. Creates the files and folders the record names. +fn plant_rocm_sdk_probe(role: Role, install_root: &Path, scripts: &Path) -> serde_json::Value { + let site_packages = site_packages(install_root); + let core = site_packages.join("_rocm_sdk_core"); + let libraries = site_packages.join(format!("_rocm_sdk_libraries_{FAMILY}")); + let devel = site_packages.join("_rocm_sdk_devel"); + let mut runtime_roots = vec![core.clone(), libraries.clone()]; + if role.includes_devel() { + runtime_roots.insert(0, devel); + } + let root = runtime_roots[0].clone(); + let mut bin_paths: Vec = runtime_roots.iter().map(|root| root.join("bin")).collect(); + bin_paths.push(scripts.to_path_buf()); + let library_paths: Vec = runtime_roots + .iter() + .flat_map(|root| [root.join("bin"), root.join("lib")]) + .collect(); + for dir in &library_paths { + std::fs::create_dir_all(dir) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", dir.display())); + } + let amdhip = library_path(&core, "amdhip64"); + let hipblas = library_path(&libraries, "hipblas"); + for file in [&amdhip, &hipblas] { + write_file(file, b"stub"); + } + serde_json::json!({ + "import_ok": true, + "rocm_sdk_version": role.version(), + "site_packages": site_packages, + "root_path": root, + "bin_path": root.join("bin"), + "cmake_path": role.includes_devel().then(|| root.join("lib").join("cmake")), + "runtime_roots": runtime_roots, + "bin_paths": bin_paths, + "library_paths": library_paths, + "default_target_family": DEVICE_TARGET, + "available_target_families": [DEVICE_TARGET], + "resolved_target_family": DEVICE_TARGET, + "packages": [], + "library_shortnames": ["amdhip64", "hipblas"], + "resolved_libraries": [ + {"shortname": "amdhip64", "paths": [amdhip]}, + {"shortname": "hipblas", "paths": [hipblas]}, + ], + "error": null, + }) +} + +/// The manifest the wheel install writes for `role`, with every field the +/// writing version serialized, and the files its readiness check asks for. +fn plant_manifest(world: &E2eWorld, role: Role, key: &str) -> (PathBuf, serde_json::Value) { + let data_dir = world.app_paths().data_dir; + let install_root = data_dir.join("runtimes").join("wheel").join(key); + let scripts = install_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); + let python = scripts.join(fixture_target_name("python")); + write_file(&python, b"stub"); + // The interpreter the venv was created from: ROCm CLI's managed Python. + let python_launcher = data_dir + .join("tools") + .join("python") + .join(fixture_target_name("python")); + let [torch, ..] = role.torch_stack(); + + let mut manifest = serde_json::json!({ + "runtime_key": key, + "runtime_id": role.runtime_id(), + "channel": role.channel(), + "format": "wheel", + "family": FAMILY, + "family_source": "host", + "version": role.version(), + "install_root": install_root, + "selected_artifact_url": role.index_url(), + "index_url": role.index_url(), + "tarball_file_name": null, + "python_launcher": python_launcher, + "python_executable": python, + "pip_cache_dir": null, + "rocm_sdk": plant_rocm_sdk_probe(role, &install_root, &scripts), + "sdk_torch": torch, + "wheel_composition": { + "source_layout_generation": role.layout_generation(), + "package_specs": role.package_specs(), + "rocm_sdk_target": DEVICE_TARGET, + }, + "read_only": false, + "imported_from": null, + "installed_at_unix_ms": match role { + Role::Rocm7 => 1_757_000_000_000u64, + Role::Rocm10Release => 1_762_000_000_000, + Role::Rocm10Nightly => 1_762_500_000_000, + }, + }); + if role != Role::Rocm7 { + let fields = manifest + .as_object_mut() + .expect("the manifest is a JSON object"); + fields.insert( + "source_layout_generation".into(), + role.layout_generation().into(), + ); + fields.insert("devel".into(), role.includes_devel().into()); + } + (install_root, manifest) +} + +/// Runs `rocm` with the startup update check disabled. `command_env` is +/// consumed by each run, so it is set again for every one. +fn run(world: &mut E2eWorld, args: &[&str]) -> (String, String, i32) { + disable_startup_update_check(world); + crate::run_rocm_with_scenario_env(world, args) +} + +/// [`run`], recording the outcome on the World for the Then steps. +fn run_recorded(world: &mut E2eWorld, args: &[&str]) { + let (stdout, stderr, rc) = run(world, args); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// [`run`], failing the step with the full diagnostic bundle unless it exits 0. +/// For actions that only set the stage or have their outcome checked on disk. +fn run_ok(world: &mut E2eWorld, args: &[&str]) { + let (stdout, stderr, rc) = run(world, args); + assert!( + rc == 0, + "{}", + cli_failure_report(args, rc, &stdout, &stderr) + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +fn string_field(value: Option<&serde_json::Value>, field: &str) -> Option { + value + .and_then(|value| value.get(field)) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) +} + +/// What the config and the active-runtime marker name now. +fn in_use(world: &E2eWorld) -> InUse { + let config = read_json_if_present(&config_path(world)); + let marker = read_json_if_present(&active_marker_path(world)); + InUse { + config_active: string_field(config.as_ref(), "active_runtime_key"), + config_previous: string_field(config.as_ref(), "previous_runtime_key"), + config_default_id: string_field(config.as_ref(), "default_runtime_id"), + marker_key: string_field(marker.as_ref(), "runtime_key"), + marker_id: string_field(marker.as_ref(), "runtime_id"), + marker_previous_key: string_field(marker.as_ref(), "previous_runtime_key"), + marker_previous_id: string_field(marker.as_ref(), "previous_runtime_id"), + } +} + +/// What both records must name while `role` is in use with `previous` as its +/// rollback target. +fn expected_in_use(world: &E2eWorld, role: Role, previous: Option) -> InUse { + let role_key = Some(key(world, role)); + let previous_key = previous.map(|previous| key(world, previous)); + let previous_id = previous.map(|previous| previous.runtime_id().to_owned()); + InUse { + config_active: role_key.clone(), + config_previous: previous_key.clone(), + config_default_id: Some(role.runtime_id().to_owned()), + marker_key: role_key, + marker_id: Some(role.runtime_id().to_owned()), + marker_previous_key: previous_key, + marker_previous_id: previous_id, + } +} + +fn activate(world: &mut E2eWorld, role: Role) { + let key = key(world, role); + run_ok(world, &["runtimes", "activate", &key]); + let in_use = in_use(world); + world + .side_by_side + .as_mut() + .expect("no side-by-side runtimes were planted in this scenario") + .activations + .push(Activation { role, in_use }); +} + +fn assert_in_use(world: &E2eWorld, role: Role, previous: Option) { + assert_eq!( + in_use(world), + expected_in_use(world, role, previous), + "the config and the active-runtime marker must name the {role:?} runtime as in use, \ + with {previous:?} as its rollback target" + ); +} + +/// Fails unless `runtime`'s folder is still there and both its install records +/// hold exactly the bytes that were planted. +fn assert_records_untouched(runtime: &PlantedRuntime) { + let role = runtime.role; + assert!( + runtime.install_root.is_dir(), + "the {role:?} runtime's folder was removed: {}", + runtime.install_root.display() + ); + for (path, planted) in [ + (runtime.registry_path.clone(), &runtime.registry_bytes), + (runtime.local_manifest_path(), &runtime.local_manifest_bytes), + ] { + let now = std::fs::read(&path).unwrap_or_else(|error| { + panic!( + "the {role:?} runtime's record {} is gone: {error}", + path.display() + ) + }); + assert!( + &now == planted, + "the {role:?} runtime's record {} was rewritten.\nplanted:\n{}\nnow:\n{}", + path.display(), + String::from_utf8_lossy(planted), + String::from_utf8_lossy(&now) + ); + } +} + +fn output_text(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +// ── Given ────────────────────────────────────────────────────────── + +#[given( + "a ROCm 7 runtime installed before the ROCm 10 layout existed, a ROCm 10 release runtime and a ROCm 10 nightly runtime, all for one GPU family" +)] +async fn plant_side_by_side_runtimes(world: &mut E2eWorld) { + let registry = registry_dir(world); + let runtimes = Role::ALL + .into_iter() + .map(|role| { + let key = role.key(); + assert_eq!( + key, + role.expected_key(), + "the {role:?} fixture no longer derives the key the production code \ + gives these inputs; update `Role::expected_key` together with \ + `wheel_runtime_key_matches_the_side_by_side_e2e_fixture` in \ + apps/rocm/src/therock.rs" + ); + let (install_root, manifest) = plant_manifest(world, role, &key); + let manifest = serde_json::to_vec_pretty(&manifest) + .expect("failed to serialize the runtime manifest"); + let registry_path = registry.join(format!("{key}.json")); + write_file(&install_root.join(".rocm-cli-runtime.json"), &manifest); + write_file(®istry_path, &manifest); + PlantedRuntime { + role, + key, + install_root, + registry_path, + registry_bytes: manifest.clone(), + local_manifest_bytes: manifest, + } + }) + .collect(); + world.side_by_side = Some(SideBySide { + runtimes, + activations: Vec::new(), + }); +} + +#[given("the nightly runtime is in use after the ROCm 10 release runtime")] +async fn nightly_in_use_after_rocm10_release(world: &mut E2eWorld) { + activate(world, Role::Rocm10Release); + activate(world, Role::Rocm10Nightly); + assert_in_use(world, Role::Rocm10Nightly, Some(Role::Rocm10Release)); +} + +// ── When ─────────────────────────────────────────────────────────── + +#[when("the user lists the installed runtimes")] +async fn list_installed_runtimes(world: &mut E2eWorld) { + run_recorded(world, &["runtimes", "list"]); +} + +#[when("the user activates each of the three runtimes in turn")] +async fn activate_each_in_turn(world: &mut E2eWorld) { + for role in Role::ALL { + activate(world, role); + } +} + +#[when("the user uninstalls the ROCm 10 release runtime")] +async fn uninstall_rocm10_release(world: &mut E2eWorld) { + let key = key(world, Role::Rocm10Release); + run_ok(world, &["runtimes", "uninstall", &key, "--yes"]); +} + +#[when("the user uninstalls the runtime in use")] +async fn uninstall_runtime_in_use(world: &mut E2eWorld) { + let in_use = in_use(world) + .config_active + .expect("the config names no runtime in use to uninstall"); + run_ok(world, &["runtimes", "uninstall", &in_use, "--yes"]); +} + +#[when("the user activates the ROCm 7 runtime")] +async fn activate_rocm7(world: &mut E2eWorld) { + activate(world, Role::Rocm7); +} + +#[when("the user activates a runtime by the name the two release runtimes share")] +async fn activate_by_shared_name(world: &mut E2eWorld) { + run_recorded(world, &["runtimes", "activate", SHARED_RELEASE_RUNTIME_ID]); +} + +// ── Then ─────────────────────────────────────────────────────────── + +#[then("all three runtimes are listed as ready, each under its own key")] +async fn all_three_listed_ready(world: &mut E2eWorld) { + let stdout = world.cli_output.clone().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "`rocm runtimes list` failed:\n{}", + output_text(world) + ); + let rows: Vec<&str> = stdout + .lines() + .filter(|line| line.contains(" runtime_id=")) + .collect(); + assert_eq!( + rows.len(), + 3, + "expected one row per planted runtime:\n{stdout}" + ); + for role in Role::ALL { + let prefix = format!("{} runtime_id={} ", key(world, role), role.runtime_id()); + let matching: Vec<&&str> = rows.iter().filter(|row| row.contains(&prefix)).collect(); + let [row] = matching.as_slice() else { + panic!("expected exactly one row for the {role:?} runtime ({prefix}):\n{stdout}"); + }; + assert!( + row.contains(" status=ready "), + "the {role:?} runtime must be listed as ready: {row}" + ); + assert!( + row.ends_with(&format!(" toolchain={}", role.toolchain())), + "the {role:?} runtime must be listed with toolchain={}: {row}", + role.toolchain() + ); + } +} + +#[then("each one becomes the runtime in use in turn")] +async fn each_becomes_in_use(world: &mut E2eWorld) { + let activations = &state(world).activations; + let order: Vec = activations + .iter() + .map(|activation| activation.role) + .collect(); + assert_eq!(order, Role::ALL, "the activations did not all run"); + let mut previous = None; + for activation in activations { + assert_eq!( + activation.in_use, + expected_in_use(world, activation.role, previous), + "activating the {:?} runtime did not make it the one in use, with {previous:?} \ + as its rollback target", + activation.role + ); + previous = Some(activation.role); + } +} + +#[then("none of the three runtimes' install records was rewritten")] +async fn no_record_rewritten(world: &mut E2eWorld) { + for runtime in &state(world).runtimes { + assert_records_untouched(runtime); + } +} + +#[then("that runtime's folder and record are gone")] +async fn rocm10_release_gone(world: &mut E2eWorld) { + let runtime = state(world).runtime(Role::Rocm10Release); + assert!( + !runtime.install_root.exists(), + "the uninstalled runtime's folder is still there: {}\n{}", + runtime.install_root.display(), + output_text(world) + ); + assert!( + !runtime.registry_path.exists(), + "the uninstalled runtime's registry entry is still there: {}", + runtime.registry_path.display() + ); +} + +#[then("the other two runtimes keep their folders and unchanged records")] +async fn others_untouched(world: &mut E2eWorld) { + let state = state(world); + for role in [Role::Rocm7, Role::Rocm10Nightly] { + assert_records_untouched(state.runtime(role)); + } +} + +#[then("the nightly runtime is still in use, with no rollback target left")] +async fn nightly_still_in_use(world: &mut E2eWorld) { + assert_in_use(world, Role::Rocm10Nightly, None); +} + +#[then("no runtime is in use")] +async fn no_runtime_in_use(world: &mut E2eWorld) { + let nothing = InUse { + config_active: None, + config_previous: None, + config_default_id: None, + marker_key: None, + marker_id: None, + marker_previous_key: None, + marker_previous_id: None, + }; + assert_eq!( + in_use(world), + nothing, + "neither the config nor the active-runtime marker may name a runtime in use or a \ + rollback target:\n{}", + output_text(world) + ); +} + +#[then("the ROCm 7 runtime is the runtime in use")] +async fn rocm7_in_use(world: &mut E2eWorld) { + assert_in_use(world, Role::Rocm7, None); +} + +#[then("the CLI refuses and names both release runtimes")] +async fn refused_naming_both_release_runtimes(world: &mut E2eWorld) { + let text = output_text(world); + assert_ne!( + world.cli_rc, + Some(0), + "a name shared by two runtimes must be refused:\n{text}" + ); + for role in [Role::Rocm7, Role::Rocm10Release] { + let key = key(world, role); + assert!( + text.contains(&key), + "the refusal must name the {role:?} runtime ({key}):\n{text}" + ); + } + let nightly = key(world, Role::Rocm10Nightly); + assert!( + !text.contains(&nightly), + "the refusal must not name the nightly runtime, which does not share the name:\n{text}" + ); + // The way out it names works: runtime-lifecycle-19's "each one becomes the + // runtime in use in turn" runs `rocm runtimes activate ` for + // each of these runtimes (`activate`, above) and checks it takes effect. + assert!( + text.contains("activate one by runtime_key"), + "the refusal must say to activate one of them by its key:\n{text}" + ); +} diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b..43eea8f86 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -5734,15 +5734,21 @@ fi ); } - /// The hermetic comfyui-01/02/03 scenarios (no GPU, no network, no - /// `e2e-test-hooks` dependency, Windows-portable since #637) must stay - /// explicitly selected into this otherwise-`@lifecycle`-only Windows lane - /// (#689) — it is required CI, unlike the self-hosted lane that otherwise - /// runs them. This checks the whole step (not just the `run: |` body, since - /// the selection lives in `env:`), so dropping the env var — or narrowing - /// its id list — fails here rather than silently reopening the coverage gap. - #[test] - fn ci_windows_lifecycle_lane_also_selects_the_hermetic_comfyui_and_vllm_refusal_scenarios() { + /// The hermetic non-`@lifecycle` scenarios this otherwise-`@lifecycle`-only + /// Windows lane selects through `E2E_ALSO_RUN_IDS` (#689): comfyui-01/02/03 + /// (Windows-portable since #637), the native-Windows vLLM refusal + /// serve-33/34/35, and the side-by-side runtime scenarios + /// runtime-lifecycle-19/20/21. None needs a GPU, the network or + /// `e2e-test-hooks`, and this is required CI, unlike the self-hosted lane + /// that otherwise runs them. This checks the whole step (not just the + /// `run: |` body, since the selection lives in `env:`) and pins the id list + /// exactly, so dropping the env var, narrowing the list or growing it + /// without updating this test fails here. It also holds the PowerShell + /// copy of the list in docs/testing.md equal to the ci.yml one; the + /// "19/20/21"-style shorthand in docs/testing.md's prose and in + /// tests/e2e-cucumber/README.md is not checked. + #[test] + fn ci_windows_lifecycle_lane_also_selects_exactly_the_hermetic_non_lifecycle_scenarios() { let ci = read_workflow("ci.yml"); let windows = job_block(&ci, "windows-release"); let lines: Vec<&str> = windows.lines().collect(); @@ -5761,24 +5767,62 @@ fi also_run_ids.len(), 1, "the Windows lifecycle lane must set env.E2E_ALSO_RUN_IDS exactly \ - once to additionally select the hermetic comfyui and vLLM-refusal scenarios, so \ - they get required-CI Windows coverage (#689):\n{step}" - ); - for id in [ - "comfyui-uv-install-failure-names-log", - "comfyui-uv-install-progress-streamed", - "comfyui-ambiguous-runtime-actionable", - // The native-Windows vLLM refusal (serve-33/34/35) is - // `@requires-os:windows` and hermetic, so this is its required lane. - "serve-vllm-refused-on-native-windows", - "serve-vllm-install-refused-on-native-windows", - "serve-vllm-install-with-runtime-refused-on-native-windows", - ] { - assert!( - also_run_ids[0].contains(id), - "the Windows lifecycle lane must select {id} via env.E2E_ALSO_RUN_IDS:\n{step}" - ); - } + once to additionally select the hermetic comfyui, vLLM-refusal and \ + side-by-side runtime scenarios, so they get required-CI Windows \ + coverage (#689):\n{step}" + ); + let selected: Vec<&str> = also_run_ids[0] + .trim_matches('"') + .split(',') + .map(str::trim) + .collect(); + assert_eq!( + selected, + [ + "comfyui-uv-install-failure-names-log", + "comfyui-uv-install-progress-streamed", + "comfyui-ambiguous-runtime-actionable", + // The native-Windows vLLM refusal (serve-33/34/35) is + // `@requires-os:windows` and hermetic, so this is its required lane. + "serve-vllm-refused-on-native-windows", + "serve-vllm-install-refused-on-native-windows", + "serve-vllm-install-with-runtime-refused-on-native-windows", + // runtime-lifecycle-19/20/21: planted side-by-side runtimes, + // hermetic, whose Windows paths only this lane exercises. + "runtime-lifecycle-side-by-side-across-packaging-change", + "runtime-lifecycle-side-by-side-uninstall-keeps-the-others", + "runtime-lifecycle-side-by-side-shared-name-refused", + ], + "the Windows lifecycle lane's env.E2E_ALSO_RUN_IDS must select exactly the \ + hermetic comfyui-01/02/03, serve-33/34/35 and runtime-lifecycle-19/20/21 \ + scenarios; update this list, docs/testing.md and \ + tests/e2e-cucumber/README.md together with ci.yml:\n{step}" + ); + + // docs/testing.md shows the same selection as a PowerShell line to run + // the lane locally; it must not drift from what CI selects. + let testing = std::fs::read_to_string(repo_root().join("docs/testing.md")) + .expect("docs/testing.md must be readable") + .replace("\r\n", "\n"); + let documented: Vec<&str> = testing + .lines() + .filter_map(|line| line.trim().strip_prefix("$env:E2E_ALSO_RUN_IDS = ")) + .collect(); + assert_eq!( + documented.len(), + 1, + "docs/testing.md must show the Windows lane's E2E_ALSO_RUN_IDS once" + ); + let documented: Vec<&str> = documented[0] + .trim_matches('"') + .split(',') + .map(str::trim) + .collect(); + assert_eq!( + documented, selected, + "docs/testing.md's $env:E2E_ALSO_RUN_IDS line must list exactly the ids \ + ci.yml's Windows lifecycle lane selects, in the same order" + ); } /// Every job in `ci.yml` as `(id, block)`, in file order. Unlike [`job_block`]