From 30a43a6d1d5681afda89d34427a42b83f8a20d30 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 30 Oct 2026 18:13:50 +0200 Subject: [PATCH 1/2] fix(sdk): install the newest stable ROCm 10 release when no version is pinned An unpinned release-channel rocm install sdk stayed on the canonical index and installed ROCm 7.14. It now resolves the newest stable version on the next layout, with that layout's Python, and still leaves pinned 7.x and nightly auto-select on the canonical stream. Signed-off-by: Roman Inflianskas --- .github/workflows/e2e-selfhosted.yml | 10 +- .github/workflows/nightly.yml | 14 +- MANIFEST.md | 15 +- README.md | 45 ++- apps/rocm/src/main.rs | 13 +- apps/rocm/src/therock.rs | 378 +++++++++++++----- apps/rocm/src/therock/version.rs | 3 +- docs/ci-hardware-testing.md | 2 +- docs/manual-testing.md | 11 +- docs/release-trust.md | 5 +- docs/testing.md | 22 +- scripts/reclaim-gpu.sh | 8 +- .../features/runtime_setup.feature | 12 +- .../features/therock_next_generation.feature | 45 ++- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 52 ++- tests/e2e-cucumber/tests/e2e/therock_steps.rs | 150 ++++++- xtask/src/workflow_contract.rs | 4 +- 17 files changed, 587 insertions(+), 202 deletions(-) diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 3a0119f900..8750170431 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -329,7 +329,7 @@ jobs: # instantly (observed on run 29320025393). Installing in place keeps the # baked paths valid, and each scenario's data/runtimes symlink resolves to # this same real tree. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build the rocm and rocmd binaries ONCE and reuse them for both the @@ -510,7 +510,7 @@ jobs: # is missing)` and fails (diagnosed on the box 2026-07-15). Installing in # place, directly into the persistent shared dir, keeps the baked paths # valid for all scenarios and for the end-of-run version probe. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite. @@ -831,7 +831,7 @@ jobs: # every later serve sees status=unusable and fails (diagnosed on the Linux # box 2026-07-15; the Windows scenario-8 cold-download failure is the same # class). Install in place so the baked paths stay valid for all scenarios. - $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm-multi-arch-v2" + $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm-next-v1" $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" # E2E_SHARED_CACHE_DIR: every Linux lane sets this; the harness's own @@ -1415,7 +1415,7 @@ jobs: # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the # runtime manifest, so installing into a per-scenario temp dir leaves # every later serve pointing at a deleted install root. - prewarm="/root/work/e2e-prewarm-multi-arch-v2" + prewarm="/root/work/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build the rocm and rocmd binaries once; reuse them for pre-warm + suite. @@ -1598,7 +1598,7 @@ jobs: # optimization — `install sdk` bakes absolute paths into the runtime # manifest, so installing anywhere temporary breaks every later serve. # See e2e-gpu for the full rationale. - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane for why the e2e-test-hooks feature must match diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e9d00ad47a..24f9a0e98b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -438,7 +438,7 @@ jobs: # The shared dir IS the pre-warm's own data/runtimes; NEVER move it — a # post-install mv invalidates the absolute paths install sdk bakes into # the runtime manifest and every serve fails instantly (run 29320025393). - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # Build both binaries once; reuse them for pre-warm + suite. @@ -529,7 +529,7 @@ jobs: # (see e2e-selfhosted.yml e2e-gpu). runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane in e2e-selfhosted.yml for why the @@ -605,7 +605,7 @@ jobs: # (see e2e-selfhosted.yml e2e-gpu). runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane in e2e-selfhosted.yml for why the @@ -687,7 +687,7 @@ jobs: # (see e2e-selfhosted.yml e2e-gpu). runner_root="$(dirname "$(dirname "$RUNNER_WORKSPACE")")" export E2E_SHARED_UV_CACHE_DIR="$runner_root/uv-cache" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane in e2e-selfhosted.yml for why the @@ -774,7 +774,7 @@ jobs: run: | export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" - prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane in e2e-selfhosted.yml for why the @@ -1021,7 +1021,7 @@ jobs: # absolute paths. On the ephemeral DevLab Dispatch pool this host is # destroyed after the job, so the paths only need to stay valid for # this run's scenarios, not across runs. - $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm-multi-arch-v2" + $prewarm = "$env:RUNNER_WORKSPACE\e2e-prewarm-next-v1" $env:E2E_SHARED_RUNTIMES_DIR = "$prewarm\data\runtimes" # See e2e-gpu-strix-windows in e2e-selfhosted.yml: the harness's own @@ -1459,7 +1459,7 @@ jobs: # correctness, not speed: `install sdk` bakes ABSOLUTE paths into the # runtime manifest, so installing into a per-scenario temp dir leaves # every later serve pointing at a deleted install root. - prewarm="/root/work/e2e-prewarm-multi-arch-v2" + prewarm="/root/work/e2e-prewarm-next-v1" export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" # See the e2e-gpu lane in e2e-selfhosted.yml for why the diff --git a/MANIFEST.md b/MANIFEST.md index 687b4493d3..c6a25dee5e 100644 --- a/MANIFEST.md +++ b/MANIFEST.md @@ -659,12 +659,17 @@ ROCm distribution. Two install formats are supported: - **Wheel format** — Python wheel packages (`rocm`, `torch`, `torchvision`, `torchaudio`) are resolved from AMD-hosted PyPI-compatible indexes and - installed via `uv` into a managed virtual environment. Release channel wheels - are served from `https://repo.amd.com/rocm/whl-multi-arch`. Nightly channel - wheels are served from `https://rocm.nightlies.amd.com/whl-multi-arch`. + installed via `uv` into a managed virtual environment. An unpinned release + install, and a pin of stable ROCm 10 or newer, resolves wheels from + `https://stable.repo.amd.com/rocm/whl-next`. A pin of an older stable release, + such as `--version 7.14.1`, uses `https://repo.amd.com/rocm/whl-multi-arch`. + Nightly channel wheels are served from + `https://rocm.nightlies.amd.com/whl-multi-arch`. - **Tarball format** — Prebuilt SDK tarballs are downloaded from AMD-hosted - artifact storage. Release channel tarballs are served from - `https://repo.amd.com/rocm/tarball/`. Nightly tarballs are served from + artifact storage. An unpinned release install, and a pin of stable ROCm 10 + or newer, uses `https://stable.repo.amd.com/rocm/core/tarball/`. A runtime + installed from `https://repo.amd.com/rocm/tarball/` stays on that catalog + when it is updated. Nightly tarballs are served from `https://rocm.nightlies.amd.com/tarball/`. The metadata index files for tarball installs are optionally verified with an RSA/SHA-256 detached signature (controlled by `ROCM_CLI_METADATA_PUBLIC_KEY_PATH` / diff --git a/README.md b/README.md index 45153581d7..7f77407963 100644 --- a/README.md +++ b/README.md @@ -252,11 +252,13 @@ This downloads TheRock ROCm wheels and a matching PyTorch stack into a managed environment. Running `rocm install sdk` creates a managed runtime alongside any legacy install. -ROCm CLI supports ROCm 7.14, 10.0, and 10.1. Without `--version`, -`rocm install sdk` currently installs ROCm 7.14. To install ROCm 10.0 or 10.1, -add `--version`. If ROCm CLI can't tell which exact GPU arch this computer has, -or you want a different one, also add `--family` with the raw `gfx` code from -`rocm examine`. For example: `--version 10.1.0 --family gfx1200`. +ROCm CLI supports ROCm 7.14 and the ROCm 10 line. Without `--version`, +`rocm install sdk` on the release channel installs the newest stable ROCm 10 +release. That release needs an exact GPU arch. If ROCm CLI can't tell which +arch this computer has, or you want a different one, add `--family` with the +raw `gfx` code from `rocm examine`. For example: `--family gfx1200`. +To install ROCm 7.14.1 instead, pass `--version 7.14.1`. `--channel nightly` +with no version stays on the nightly stream. Running `rocm install sdk` when a managed runtime is already the active default asks first, because the new install takes over as the active default; see @@ -455,9 +457,10 @@ rocm update [--apply] [--runtime KEY] [--activate] [--dry-run] ``` `install sdk` downloads TheRock ROCm wheels into a Python environment managed -by ROCm CLI. ROCm CLI supports ROCm 7.14, 10.0, and 10.1. Without `--version`, -it currently installs ROCm 7.14. To install 10.0 or 10.1, follow the steps under -ROCm 10 and newer, below. +by ROCm CLI. On the release channel, with no `--version`, it installs the +newest stable ROCm 10 release. Pass `--version 7.14.1` to install that older +release instead. `--channel nightly` with no version stays on the nightly +stream. ROCm 10 needs an exact GPU arch; see ROCm 10 and newer, below. To remove ROCm CLI and what it manages, see [Logs and cleanup](#logs-and-cleanup). @@ -531,14 +534,16 @@ active default runtime, not about what a named prefix loses. #### ROCm 10 and newer -ROCm 10 and newer ship from a different source layout. You opt in by pinning the -version with `--version`. These versions select their device payload by exact -GPU arch, so when ROCm CLI can't resolve one for this computer (it detects only -a family of GPUs, or none), or you want to target a different GPU, also pass -`--family` with the raw `gfx` code instead of a family label: +ROCm 10 and newer ship from a different source layout than ROCm 7. On the +release channel, `rocm install sdk` with no `--version` installs the newest +stable release from that layout and prepares Python 3.14 for it. These +versions select their device payload by exact GPU arch, so when ROCm CLI can't +resolve one for this computer (it detects only a family of GPUs, or none), or +you want to target a different GPU, also pass `--family` with the raw `gfx` +code instead of a family label: ``` -rocm install sdk --version 10.1.0 --family gfx1200 --dry-run +rocm install sdk --family gfx1200 --dry-run ``` A family label such as `--family gfx120X-all` is rejected for those versions @@ -565,10 +570,14 @@ dependency metadata alongside the pinned torchaudio. Before it creates or changes a managed runtime, it validates that every selected framework package carries the same ROCm build identifier. -Nothing about this happens on its own. Without a `--version` of 10 or newer, -`install sdk` resolves the same release and nightly sources as before. It doesn't -quietly retry against the ROCm 10 sources when a lookup finds nothing; it tells -you what it couldn't find instead. +An older release stays on the earlier source. `rocm install sdk --version 7.14.1` +installs 7.14.1 from that source and uses Python 3.12. `--channel nightly` with +no version stays on the nightly stream. Pinning a ROCm 10 version still works, +for example `rocm install sdk --version 10.1.0 --family gfx1200`. Updating a +runtime keeps the source layout that installed it, so `rocm update` does not +move a ROCm 7 runtime onto ROCm 10. The install does not quietly retry the +other source when a lookup finds nothing; it tells you what it couldn't find +instead. #### Driver installation diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 1de8b1ea2f..d8ffb452dd 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -777,10 +777,16 @@ enum BenchCommand { #[derive(Subcommand, Debug)] enum InstallTarget { /// Install TheRock ROCm wheels into a Python folder managed by ROCm CLI. + /// + /// With no `--version` on the release channel, this installs the newest + /// stable ROCm 10 release and needs an exact GPU arch (`--family gfx1200`). + /// `--channel nightly` with no version stays on the nightly index. + /// `--version 7.14.1` installs that release from the earlier index. #[command(after_help = "EXAMPLES:\n \ rocm install sdk\n \ +rocm install sdk --family gfx1200 --dry-run\n \ +rocm install sdk --version 7.14.1 --family gfx110X-all --dry-run\n \ rocm install sdk --channel nightly --build-date 2025-01-15\n \ -rocm install sdk --family gfx110X-all --dry-run\n \ rocm install sdk --devel")] Sdk { /// Package channel to install, such as release or nightly. @@ -800,7 +806,10 @@ rocm install sdk --devel")] build_date: Option, /// TheRock GPU package family to install, such as gfx110X-all. /// - /// A GPU target ROCm publishes no build for is refused rather than + /// ROCm 10, including `rocm install sdk` with no `--version` on the + /// release channel, rejects a family label and asks for an exact arch + /// such as `--family gfx1200`. A GPU target ROCm publishes no build + /// for is refused rather than /// mapped to a near neighbour: `--family gfx90c` - the Vega iGPU in /// Renoir, Lucienne, Cezanne and Barcelo APUs - fails and installs /// nothing, instead of substituting another family. A plain `rocm diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index d391c1186a..6009b2db6f 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -82,9 +82,9 @@ const THEROCK_SOURCE_LAYOUT_GENERATION: &str = "multi-arch-v2"; /// so a later update reads back the layout that produced the runtime instead of /// guessing it from the version. const THEROCK_NEXT_LAYOUT_GENERATION: &str = "next-v1"; -/// The first ROCm major version published only in the next layout. A pin at or -/// past it is the one request the canonical stream provably cannot serve, and -/// therefore the only thing that selects [`SourceLayout::Next`]. +/// The first ROCm major version published only in the next layout. A stable pin +/// at or past it selects [`SourceLayout::Next`], and so does an unpinned +/// release-channel install, which resolves the newest stable version there. const THEROCK_NEXT_MIN_MAJOR: u32 = 10; const DEFAULT_MANAGED_PYTHON_VERSION: &str = "3.12"; /// Interpreter `uv` provisions for a [`SourceLayout::Next`] install. See @@ -101,10 +101,13 @@ enum TheRockChannel { /// Which published hosting layout a resolution reads from. /// /// Orthogonal to [`TheRockChannel`]. `Canonical` is the release/nightly stream -/// the CLI has always installed and remains the answer for every request that -/// does not explicitly ask for something else. `Next` is ROCm 10's separately -/// hosted layout — an extension, never a fallback: nothing degrades into it and -/// nothing degrades out of it, so a canonical install resolves exactly the URLs, +/// the CLI has always installed. It remains the answer for nightly auto-select, +/// for any pin that is not a stable ROCm >= [`THEROCK_NEXT_MIN_MAJOR`] version, +/// and for an update of a runtime that was installed from it. `Next` is ROCm +/// 10's separately hosted layout. An unpinned release-channel install reads it +/// for the newest stable version, and so does an explicit stable pin at +/// ROCm >= [`THEROCK_NEXT_MIN_MAJOR`]. It is never a fallback: a request that +/// cannot use it refuses, and a canonical install still resolves the URLs, /// device target, and package specs it resolved before the layout existed. #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum SourceLayout { @@ -306,35 +309,57 @@ fn next_layout_requested(selector: &RuntimeVersionSelector) -> bool { }) } +/// Whether this request reads [`SourceLayout::Next`] before arch and channel +/// checks. An explicit stable pin at ROCm >= [`THEROCK_NEXT_MIN_MAJOR`] does, +/// and so does an unpinned release-channel install. Nightly auto-select, a +/// build date, an unparseable pin, a prerelease, and any stable pin below that +/// major stay on the canonical stream. +fn selects_next_layout( + channel: TheRockChannel, + version_selector: Option<&RuntimeVersionSelector>, +) -> bool { + if version_selector.is_some_and(next_layout_requested) { + return true; + } + version_selector.is_none() && matches!(channel, TheRockChannel::Release) +} + /// The layout an install must read from, and the refusal when it cannot. /// -/// Only an explicit pin at ROCm >= [`THEROCK_NEXT_MIN_MAJOR`] selects `Next`, -/// because that is the one request the canonical stream provably cannot serve. -/// No selector, a build date, an unparseable pin, or an older pin all stay -/// canonical, so nothing that resolves today starts resolving somewhere else. -/// A pin that does reach `Next` still needs an exact arch, because that layout -/// picks its device payload by exact arch and a grouped family names none. +/// [`selects_next_layout`] decides the route. A pin that names a stable +/// ROCm >= [`THEROCK_NEXT_MIN_MAJOR`] version on a channel other than release +/// is refused: that layout has no nightly stream. Every route onto `Next` +/// needs an exact arch, because that layout picks its device payload by exact +/// arch and a grouped family names none. A missing arch stops here. It does +/// not fall back to the canonical stream. fn select_source_layout( channel: TheRockChannel, family_resolution: &FamilyResolution, version_selector: Option<&RuntimeVersionSelector>, ) -> Result { - let Some(selector) = version_selector.filter(|selector| next_layout_requested(selector)) else { + if !selects_next_layout(channel, version_selector) { return Ok(SourceLayout::Canonical); - }; - let RuntimeVersionSelector::Version(version) = selector else { - return Ok(SourceLayout::Canonical); - }; - let major = parse_version(version).map_or(THEROCK_NEXT_MIN_MAJOR, |parsed| parsed.major); - if !matches!(channel, TheRockChannel::Release) { + } + let pinned = version_selector.and_then(|selector| match selector { + RuntimeVersionSelector::Version(version) => Some(version.as_str()), + RuntimeVersionSelector::BuildDate(_) => None, + }); + let major = pinned + .and_then(parse_version) + .map_or(THEROCK_NEXT_MIN_MAJOR, |parsed| parsed.major); + if let Some(version) = pinned.filter(|_| !matches!(channel, TheRockChannel::Release)) { bail!( "ROCm {major} and newer is published only on the release channel; re-run `rocm install sdk --channel release --version {version}`" ); } if family_resolution.raw_arch.is_none() { + let example = match pinned { + Some(version) => format!("`rocm install sdk --version {version} --family gfx1200`"), + None => "`rocm install sdk --family gfx1200`".to_owned(), + }; bail!( "installing ROCm {major} requires an exact GPU arch, but the resolved target family `{}` names a group of them.\n\ - ROCm {major} selects its device payload by exact arch, so re-run with the one this host has, for example `rocm install sdk --version {version} --family gfx1200`.\n\n{}", + ROCm {major} selects its device payload by exact arch, so re-run with the one this host has, for example {example}.\n\n{}", family_resolution.family, detect_host_gpu_diagnostics() ); @@ -778,9 +803,11 @@ pub(crate) fn install_sdk( ), "tarball" => { // A tarball catalog lists whole archives, not a resolvable package - // graph, so there is nothing to pin a version against — except the - // ROCm 10 catalog, which is reachable *only* by naming a version, - // because nothing else distinguishes it from the canonical one. + // graph, so there is nothing to pin a version against on the + // canonical stream. The next catalog is reachable without a pin on + // the release channel (it installs the newest stable there) and by + // naming a stable ROCm >= THEROCK_NEXT_MIN_MAJOR version; any other + // selector still has to use `--format wheel`. if let Some(selector) = version_selector.as_ref() && !next_layout_requested(selector) { @@ -1454,33 +1481,38 @@ fn install_wheel_runtime( // does internally. let recovered_family_override = family_override .map(|family| family_override_or_recovered_arch(family, device_target_override)); - // Fail fast on a request `resolve_pip_runtime` could only ever reject (wrong - // channel, or a grouped family with no exact arch) before paying for Python - // provisioning below, which can mean a real download for a layout this host - // or channel can never actually use. - select_source_layout( - channel, - &resolve_family(paths, recovered_family_override.as_deref())?, - version_selector, - )?; + // Fail fast on a fresh request `resolve_pip_runtime` could only ever reject + // (wrong channel, or a grouped family with no exact arch) before paying for + // Python provisioning below, which can mean a real download for a layout + // this host or channel can never actually use. An update already recorded + // its layout; re-deciding from the absent selector would treat a canonical + // runtime's refresh as an unpinned release install. + if layout_override.is_none() { + select_source_layout( + channel, + &resolve_family(paths, recovered_family_override.as_deref())?, + version_selector, + )?; + } // The interpreter has to be picked before `resolve_pip_runtime`, which // selects wheels using that interpreter's own tags. `requested_source_layout` - // only answers from the *request*: an explicit stable ROCm 10+ pin already - // routes to `Next` and gets `cp314` that way, but a plain auto-select or a - // nightly build-date/prerelease pin always reports `Canonical`, even once - // the canonical/nightly feed's own "latest" has rotated past that major on - // its own, with no `Next` routing involved. `canonical_auto_select_needs_next_python` + // answers from the *request*: an unpinned release install and an explicit + // stable ROCm 10+ pin both route to `Next` and get `cp314` that way. A + // nightly build-date/prerelease pin still reports `Canonical`, even once + // the nightly feed's own "latest" has rotated past that major on its own, + // with no `Next` routing involved. `canonical_auto_select_needs_next_python` // peeks the channel's own `rocm` listing to catch that case. - let python_requirement = match requested_source_layout(layout_override, version_selector) { - SourceLayout::Next => python_requirement(SourceLayout::Next), - SourceLayout::Canonical => { - if canonical_auto_select_needs_next_python(paths, channel, version_selector)? { - python_requirement(SourceLayout::Next) - } else { - python_requirement(SourceLayout::Canonical) + let python_requirement = + match requested_source_layout(layout_override, channel, version_selector) { + SourceLayout::Next => python_requirement(SourceLayout::Next), + SourceLayout::Canonical => { + if canonical_auto_select_needs_next_python(paths, channel, version_selector)? { + python_requirement(SourceLayout::Next) + } else { + python_requirement(SourceLayout::Canonical) + } } - } - }; + }; progress_line(format!( "Checking Python for the ROCm install; if needed, ROCm CLI will prepare Python {}.", managed_python_version(python_requirement) @@ -2725,10 +2757,10 @@ fn resolve_pip_runtime_with_timeout( })?; let published_device_targets = parse_aggregate_device_targets(&root_html); // The canonical stream validates against what this host reports, exactly as - // before. The next layout is reachable only through an explicit pin that - // already carried a validated exact arch, and that arch — not a second probe - // of whatever card happens to be plugged in — is what the pin asked to - // install for. + // before. The next layout is reached only after `select_source_layout` has + // already validated an exact arch on the family resolution, and that arch — + // not a second probe of whatever card happens to be plugged in — is what the + // install asked for. let detected_target = match layout { SourceLayout::Canonical => detect_host_gfx_target(), SourceLayout::Next => family_resolution.raw_arch.clone(), @@ -3014,7 +3046,10 @@ fn tarball_family_token(layout: SourceLayout, family: &str) -> &str { /// all, and whose mtime is *later* than the archive they shadow, so an unfiltered /// "newest wins" would pick the wrong file. Requiring a parseable stable version /// excludes them without inventing a second grammar for the canonical catalogs, -/// which publish no such siblings and have never been held to one. +/// which publish no such siblings and have never been held to one. Among the +/// stables that remain, the release channel orders by version, so a republished +/// older archive cannot outrank a newer stable version. Nightly keeps upload +/// time as the first key. fn select_tarball_candidate( files: &[TarballIndexFile], channel: TheRockChannel, @@ -3045,11 +3080,20 @@ fn select_tarball_candidate( }) .collect::>(); candidates.sort_by(|left, right| { - left.0 - .mtime - .partial_cmp(&right.0.mtime) - .unwrap_or(Ordering::Equal) - .then_with(|| compare_version_strings(&left.1, &right.1)) + if matches!(channel, TheRockChannel::Release) { + compare_version_strings(&left.1, &right.1).then_with(|| { + left.0 + .mtime + .partial_cmp(&right.0.mtime) + .unwrap_or(Ordering::Equal) + }) + } else { + left.0 + .mtime + .partial_cmp(&right.0.mtime) + .unwrap_or(Ordering::Equal) + .then_with(|| compare_version_strings(&left.1, &right.1)) + } }); candidates.pop() } @@ -4492,11 +4536,11 @@ struct PythonRequirement { /// Keyed on the layout rather than on a resolved ROCm version because the /// interpreter has to be chosen *before* resolution: `resolve_pip_runtime` picks /// wheels using the interpreter's own tags, so the resolved version is not -/// available yet. The layout is an honest stand-in only when it was decided -/// from an explicit stable pin (`next_layout_requested`); a fresh install with -/// no pin, or a nightly build-date/prerelease pin, needs -/// [`canonical_auto_select_needs_next_python`] to check whether the channel's -/// own "latest" has rotated past [`THEROCK_NEXT_MIN_MAJOR`] on its own. +/// available yet. The layout is decided by [`requested_source_layout`] (an +/// unpinned release install or an explicit stable ROCm >= 10 pin). A nightly +/// build-date/prerelease pin that stays on `Canonical` still needs +/// [`canonical_auto_select_needs_next_python`] when that feed's own "latest" +/// has rotated past [`THEROCK_NEXT_MIN_MAJOR`] on its own. const fn python_requirement(layout: SourceLayout) -> PythonRequirement { match layout { SourceLayout::Next => PythonRequirement { @@ -4512,32 +4556,35 @@ const fn python_requirement(layout: SourceLayout) -> PythonRequirement { /// The layout an install or update will read from, as far as is knowable before /// an interpreter has been chosen. An update passes its manifest's recorded -/// layout; a fresh install passes its request and lets the pin decide. +/// layout; a fresh install passes its channel and selector and lets +/// [`selects_next_layout`] decide. fn requested_source_layout( layout_override: Option, + channel: TheRockChannel, version_selector: Option<&RuntimeVersionSelector>, ) -> SourceLayout { match layout_override { Some(layout) => layout, - None if version_selector.is_some_and(next_layout_requested) => SourceLayout::Next, + None if selects_next_layout(channel, version_selector) => SourceLayout::Next, None => SourceLayout::Canonical, } } -/// Whether a `Canonical`-layout install (auto-select, or a nightly +/// Whether a `Canonical`-layout install (nightly auto-select, or a nightly /// build-date/prerelease pin) will actually resolve a ROCm build at major >= /// [`THEROCK_NEXT_MIN_MAJOR`], regardless of [`VersionStage`]. /// -/// `requested_source_layout` only answers from the shape of the request: an -/// explicit *stable* pin at ROCm >= `THEROCK_NEXT_MIN_MAJOR` routes to `Next` -/// and gets `cp314` that way, but the canonical/nightly feed can resolve a -/// ROCm >= `THEROCK_NEXT_MIN_MAJOR` build (stable or prerelease) on its own, -/// with no `Next` routing involved — and the `cp314` requirement is a property -/// of the build itself, not of which index served it. Fetching the channel's -/// own interpreter-agnostic `rocm` listing ahead of interpreter selection is -/// the only way to know; `resolve_pip_runtime` re-fetches the same URL for the -/// real resolution afterwards and shares this call's cache entry, so this -/// costs no extra network round trip. +/// `requested_source_layout` answers from the shape of the request: an +/// unpinned release install and an explicit *stable* pin at ROCm >= +/// `THEROCK_NEXT_MIN_MAJOR` both route to `Next` and get `cp314` that way, but +/// the nightly feed can resolve a ROCm >= `THEROCK_NEXT_MIN_MAJOR` build +/// (stable or prerelease) on its own, with no `Next` routing involved — and +/// the `cp314` requirement is a property of the build itself, not of which +/// index served it. Fetching the channel's own interpreter-agnostic `rocm` +/// listing ahead of interpreter selection is the only way to know; +/// `resolve_pip_runtime` re-fetches the same URL for the real resolution +/// afterwards and shares this call's cache entry, so this costs no extra +/// network round trip. fn canonical_auto_select_needs_next_python( paths: &AppPaths, channel: TheRockChannel, @@ -5194,22 +5241,16 @@ mod tests { } } - /// Everything a user can ask for today must keep resolving the canonical - /// stream. The next layout is an extension, so no existing request may start - /// answering from somewhere else. + /// A build date, an older stable pin, and a string that is not a version + /// stay on the canonical stream. An unpinned nightly install does too. #[test] - fn only_a_rocm_10_pin_leaves_the_canonical_layout() { + fn selectors_that_are_not_stable_rocm_10_stay_canonical() { let exact = family_for_test("gfx120X-all", Some("gfx1200")); let build_date = RuntimeVersionSelector::build_date("2026-06-05").unwrap(); - let older_pin = RuntimeVersionSelector::version("7.14.0").unwrap(); + let older_pin = RuntimeVersionSelector::version("7.14.1").unwrap(); let unparseable = RuntimeVersionSelector::version("main").unwrap(); - for selector in [ - None, - Some(&build_date), - Some(&older_pin), - Some(&unparseable), - ] { + for selector in [Some(&build_date), Some(&older_pin), Some(&unparseable)] { assert_eq!( select_source_layout(TheRockChannel::Release, &exact, selector).unwrap(), SourceLayout::Canonical, @@ -5217,13 +5258,59 @@ mod tests { ); } - let pin = RuntimeVersionSelector::version("10.0.0").unwrap(); + assert_eq!( + select_source_layout(TheRockChannel::Nightly, &exact, None).unwrap(), + SourceLayout::Canonical, + "an unpinned nightly install stays on the nightly canonical stream" + ); + + let pin = RuntimeVersionSelector::version("10.1.0").unwrap(); assert_eq!( select_source_layout(TheRockChannel::Release, &exact, Some(&pin)).unwrap(), SourceLayout::Next ); } + /// Reverting the unpinned release route to `Canonical` fails this test. + /// The arch is exact (`gfx1151`), so the only way the result is `Canonical` + /// is the default route itself. + #[test] + fn unpinned_release_selects_the_next_layout() { + let exact = family_for_test("gfx115X-all", Some("gfx1151")); + + assert_eq!( + select_source_layout(TheRockChannel::Release, &exact, None).unwrap(), + SourceLayout::Next + ); + assert_eq!( + requested_source_layout(None, TheRockChannel::Release, None), + SourceLayout::Next + ); + assert_eq!( + python_requirement(requested_source_layout(None, TheRockChannel::Release, None)) + .version, + "3.14" + ); + } + + /// A grouped family on an unpinned release install stops. It does not fall + /// back to the canonical stream, which would install without an exact arch. + #[test] + fn unpinned_release_without_an_exact_arch_refuses_instead_of_falling_back() { + let grouped = family_for_test("gfx120X-all", None); + + let error = select_source_layout(TheRockChannel::Release, &grouped, None) + .expect_err("a grouped family cannot select a device payload") + .to_string(); + + assert!(error.contains("requires an exact GPU arch"), "{error}"); + assert!(error.contains("--family gfx1200"), "{error}"); + assert!( + !error.contains("--version"), + "an unpinned refusal must not invent a version pin: {error}" + ); + } + /// A grouped family names no single arch, and the next layout has no /// grouped device payload to fall back to, so the refusal has to name the /// flag and a value that would work. @@ -7321,31 +7408,126 @@ mod tests { } /// The requirement is read before `resolve_pip_runtime` runs, so it has to - /// be derivable from the request alone. Only an explicit stable ROCm >= 10 - /// pin reaches the next layout; a build-date pin, an older pin, or no - /// selector at all must stay canonical so nothing that installs today - /// starts demanding a different interpreter. + /// be derivable from the request alone. An unpinned release install and an + /// explicit stable ROCm >= 10 pin reach the next layout. Nightly + /// auto-select, a build-date pin, and an older pin stay canonical. #[test] fn requested_source_layout_reads_the_request_not_the_resolution() { let next_pin = RuntimeVersionSelector::Version("10.1.0".to_owned()); assert_eq!( - requested_source_layout(None, Some(&next_pin)), + requested_source_layout(None, TheRockChannel::Release, Some(&next_pin)), SourceLayout::Next ); + assert_eq!( + python_requirement(requested_source_layout( + None, + TheRockChannel::Release, + Some(&next_pin) + )) + .version, + "3.14" + ); - let old_pin = RuntimeVersionSelector::Version("7.2.3".to_owned()); + let old_pin = RuntimeVersionSelector::Version("7.14.1".to_owned()); assert_eq!( - requested_source_layout(None, Some(&old_pin)), + requested_source_layout(None, TheRockChannel::Release, Some(&old_pin)), + SourceLayout::Canonical + ); + assert_eq!( + python_requirement(requested_source_layout( + None, + TheRockChannel::Release, + Some(&old_pin) + )) + .version, + "3.12" + ); + // A two-component selector does not parse as stable >= 10. + let line_pin = RuntimeVersionSelector::Version("10.1".to_owned()); + assert_eq!( + requested_source_layout(None, TheRockChannel::Release, Some(&line_pin)), SourceLayout::Canonical ); - assert_eq!(requested_source_layout(None, None), SourceLayout::Canonical); - // An update supplies its manifest's layout, which wins over the (absent) - // selector so the runtime keeps the interpreter its stream needs. assert_eq!( - requested_source_layout(Some(SourceLayout::Next), None), + requested_source_layout(None, TheRockChannel::Release, None), SourceLayout::Next ); + assert_eq!( + requested_source_layout(None, TheRockChannel::Nightly, None), + SourceLayout::Canonical + ); + assert_eq!( + python_requirement(requested_source_layout(None, TheRockChannel::Nightly, None)) + .version, + "3.12" + ); + + // An update supplies its manifest's layout, which wins over the absent + // selector so a canonical runtime is not moved onto the next layout. + // `multi-arch-v2` is the generation those manifests record. + let recorded_canonical = SourceLayout::from_generation(Some("multi-arch-v2")).unwrap(); + assert_eq!(recorded_canonical, SourceLayout::Canonical); + assert_eq!( + requested_source_layout(Some(recorded_canonical), TheRockChannel::Release, None), + SourceLayout::Canonical, + "updating a multi-arch-v2 runtime must stay on the canonical stream" + ); + assert_eq!( + requested_source_layout(Some(SourceLayout::Next), TheRockChannel::Release, None), + SourceLayout::Next + ); + } + + /// Release auto-select takes the highest stable version. A prerelease + /// newer than that stable, and an older stable, both lose. + #[test] + fn release_auto_select_picks_the_newest_stable_not_a_newer_prerelease() { + let versions = vec![ + "10.0.0".to_owned(), + "10.2.0rc1".to_owned(), + "10.1.0".to_owned(), + "10.1.1a20261030".to_owned(), + ]; + + assert_eq!( + select_rocm_version(TheRockChannel::Release, &versions, None).as_deref(), + Some("10.1.0") + ); + } + + /// Same rule on the tarball catalog: a later mtime on an older stable, or + /// on a prerelease, does not beat the newest stable version. + #[test] + fn unpinned_next_tarball_picks_the_newest_stable_not_a_later_prerelease() { + let platform = platform_tarball_token(); + let prefix = format!("therock-dist-{platform}-gfx120X-all-"); + let files = vec![ + TarballIndexFile { + name: format!("{prefix}10.0.0.tar.gz"), + mtime: 30.0, + }, + TarballIndexFile { + name: format!("{prefix}10.1.0.tar.gz"), + mtime: 10.0, + }, + TarballIndexFile { + name: format!("{prefix}10.2.0rc1.tar.gz"), + mtime: 40.0, + }, + ]; + + let (file, version) = select_tarball_candidate( + &files, + TheRockChannel::Release, + SourceLayout::Next, + "gfx120X-all", + None, + ) + .expect("the newest stable archive must be selectable"); + + assert_eq!(version, "10.1.0"); + assert!(file.name.ends_with("10.1.0.tar.gz"), "{}", file.name); } /// A tag mismatch and a broken venv are different failures with different diff --git a/apps/rocm/src/therock/version.rs b/apps/rocm/src/therock/version.rs index 444d7798fe..a32ee9890d 100644 --- a/apps/rocm/src/therock/version.rs +++ b/apps/rocm/src/therock/version.rs @@ -70,7 +70,8 @@ pub(super) enum VersionStage { /// Total order over published version strings — the total order used for /// sorting candidates in the `therock` module's "which build does the user get?" /// decisions. It is not always the first key: [`super::select_tarball_candidate`] -/// orders by upload time first and uses this only to break ties. +/// orders a nightly catalog by upload time first and uses this only to break +/// ties. The release channel orders by this comparison first. /// /// Ordering is PEP 440's, because these strings come from a Python simple /// index: within one release `7.0.0.dev1` < `7.0.0a1` < `7.0.0b1` < diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 93384d6f85..314b4dc751 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -351,7 +351,7 @@ pre-warmed managed runtime tree (`E2E_SHARED_RUNTIMES_DIR`), so a multi-GiB lives on the runner's workspace — persistent across runs on the long-lived `native`-labeled hosts, rebuilt by the prewarm step below on each ephemeral DevLab Dispatch pool run — survives `git clean`, and is namespaced -by source-layout generation (`e2e-prewarm-multi-arch-v2`) so a branch using a new +by source-layout generation (`e2e-prewarm-next-v1`) so a branch using a new package layout cannot poison the cache consumed by code that only understands the previous layout. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 1b50a14e1f..044af68fb8 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -156,8 +156,9 @@ Expected result: `--approve-replacing-active-default` as the flag to add. - With `--approve-replacing-active-default`, the install proceeds without asking and prints a line crediting that flag by name — not `--yes`. -- `rocm install sdk ... --dry-run` never prompts or refuses, whatever the - active default is: the preview stops before the gate. +- `rocm install sdk ... --dry-run` never prompts about the active default: the + preview stops before that gate. A release install with no exact GPU arch + still refuses, including `--dry-run`. - rocm-cli creates or reuses a rocm-cli managed Python venv. - pip installs pinned `rocm`, `torch`, and `torchvision` requirements with exactly one `device-` extra (`rocm` also requests @@ -167,8 +168,10 @@ Expected result: hardware-verified build for the ROCm line (the vLLM ROCm 10.x row's exact torchaudio pin, when that row has one) or the newest published one, which it marks as not hardware-verified (a real install also logs that as a `Note:`). - On a host with no detectable AMD GPU the preview reports `device_target: undetermined` and a real install refuses - rather than pulling every published device payload. + On a host with no detectable AMD GPU, an unpinned release install refuses + before a plan and asks for an exact `--family`. A nightly preview, or a pin + of a ROCm 7 release, reports `device_target: undetermined`, and a real + install of those refuses rather than pulling every published device payload. - rocm-cli chooses the newest exact ROCm build suffix common to the SDK package and the PyTorch stack for the current Python/platform wheel tags, then pins all four packages in one pip transaction. diff --git a/docs/release-trust.md b/docs/release-trust.md index b5d4429b9b..d83715a884 100644 --- a/docs/release-trust.md +++ b/docs/release-trust.md @@ -253,8 +253,9 @@ ROCM_CLI_THEROCK_NEXT_TARBALL_BASE ``` The last two address the ROCm 10 ("next") source layout, which is a layout -rather than a channel: it has no nightly counterpart and is never resolved -unless an install explicitly pins a ROCm version of 10 or newer. +rather than a channel: it has no nightly counterpart. An unpinned release +install resolves it, and so does an explicit pin of a stable ROCm version of +10 or newer. `--channel nightly` with no version stays on the nightly index. None of these take effect unless `ROCM_CLI_THEROCK_ALLOW_BASE_OVERRIDE` is also set to a non-empty value. Without that gate, a stray override left in a diff --git a/docs/testing.md b/docs/testing.md index 63832db1e5..d5ed1bf5ca 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -451,9 +451,10 @@ rocm install sdk --channel release --format wheel --approve-replacing-active-def rocm install sdk --channel release --format wheel < /dev/null # expect the refusal ``` -The preview path is unaffected: `--dry-run` returns before the gate is -consulted, so `rocm install sdk --channel release --format wheel --dry-run` -never prompts and never refuses, whatever the active default is. +The preview path is unaffected by the active-default gate: `--dry-run` returns +before that gate is consulted, so it never prompts about replacing the active +default. A release install that has no exact GPU arch still refuses, including +`--dry-run`, and names `--family` with an exact `gfx` code. Then it verifies: @@ -468,11 +469,16 @@ Then it verifies: extra (`rocm` also requests `libraries`), plus pinned `torchaudio`, and that the toolchain is not planned for unless asked (pass the script `--devel` to check the opt-in path instead, which adds `devel` to the `rocm` extras) -- on a host with no detectable AMD GPU the preview reports `device_target: - undetermined` and renders the device extra as a placeholder; a real install - refuses rather than falling back to every published device payload -- package selection uses the newest exact ROCm build suffix common to the SDK - package and the PyTorch stack for the current Python/platform wheel tags +- on a host with no detectable AMD GPU, an unpinned release install refuses + before a plan, including `--dry-run`, and asks for `--family` with an exact + `gfx` code. It does not fall back to the ROCm 7 index. A nightly preview, or + a pin of a ROCm 7 release, still reports `device_target: undetermined` and + renders the device extra as a placeholder; a real install of those refuses + rather than falling back to every published device payload +- package selection on an unpinned release install takes the newest stable + ROCm version published on the ROCm 10 index (a newer prerelease does not + win), then pins `rocm`, `torch`, `torchvision`, and `torchaudio` for that + version - `python -m rocm_sdk version` - `python -m rocm_sdk targets` - runtime-only TheRock wheel discovery through diff --git a/scripts/reclaim-gpu.sh b/scripts/reclaim-gpu.sh index 09a89d2e26..aaa7c364ce 100755 --- a/scripts/reclaim-gpu.sh +++ b/scripts/reclaim-gpu.sh @@ -710,7 +710,7 @@ self_test() { TERM_GRACE_SECS=2 # The real shape: lemonade's engine binary inside the shared pre-warm runtime. - prewarm_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" + prewarm_decoy="${tmp}/e2e-prewarm-next-v1/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9752/llama-server" # A manual-testing serve on a shared runner: an engine, but no E2E root. workload_decoy="${tmp}/workload/manual-serve/llama-server" # An E2E root with NO engine marker — the suite's own test binary under @@ -720,7 +720,7 @@ self_test() { # Same shape as the pre-warm decoy but ignores SIGTERM, so the escalation this # script adds is reached. Without it, TERM alone ends every decoy and both the # SIGKILL block and the `kill -TERM` call can be removed with the test green. - stubborn_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" + stubborn_decoy="${tmp}/e2e-prewarm-next-v1/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9753/llama-server" # Matches the rule in full — an E2E root AND an engine marker — but lies # OUTSIDE the scope. It is the only fixture the SELFTEST_SCOPE filter can be # observed doing anything to, and therefore the only reason check 7 can fail. @@ -729,7 +729,7 @@ self_test() { # two bytes the pidcmdline record format is built out of. Without # a fixture whose command line contains them, the format is only ever exercised # on strings that cannot break it. - delimiter_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" + delimiter_decoy="${tmp}/e2e-prewarm-next-v1/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9754/llama-server" # Each pid is registered with the EXIT trap the moment it exists, for the same # reason the scratch-dir traps above are widened one statement at a time: @@ -1293,7 +1293,7 @@ self_test() { # that survived a reclaim, and a check driven from a pre-reclaim pid # would pass against an assert_clear that merely replayed check 10's # selection instead of taking its own. - late_decoy="${tmp}/e2e-prewarm-multi-arch-v2/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9755/llama-server" + late_decoy="${tmp}/e2e-prewarm-next-v1/data/runtimes/wheel/release-wheel-multi-arch-7-14-1-deadbeef/engines/lemonade/runtime/bin/llamacpp/rocm-stable/llama-b9755/llama-server" late_pid="$(spawn_decoy "${late_decoy}")" selftest_track_decoy "${late_pid}" decoy_pids+=("${late_pid}") diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 2ebd4fe962..2ceb53333e 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -160,8 +160,8 @@ Feature: Runtime configuration # resolver read a per-family index, `repo.amd.com/rocm/whl/{family}`, which is # frozen at 7.13.0 and has no device payloads at all, so every release install # got a stale SDK and a bare `rocm[libraries,devel]` — no GPU backend in it. - # Both halves are fixed by the same canonical model: one flat aggregate index - # for the channel, and the device payload for the chip this host actually has. + # An unpinned release install now reads the ROCm 10 index and still asks for + # the device payload of the chip this host actually has. # # The obvious regression test for that history is the one that does not work. # Asserting the broken per-family URL is *absent* passes with the bug present: @@ -181,11 +181,13 @@ Feature: Runtime configuration # needs a detected chip name and never opens the device. It therefore runs on # WSL hosts that can read the Windows-side target before ROCm passthrough is # ready, while mock hosts with no target skip it. - @id:runtime-resolve-canonical-release @requires-gfx-target - Scenario: runtime-09 - Previewing a release SDK install resolves the canonical aggregate for this GPU + # Needs a detected chip (`@requires-gfx-target`), not a full GPU open. It + # dry-runs against the live ROCm 10 index, so it does not run on a mock host. + @id:runtime-resolve-next-release @requires-gfx-target + Scenario: runtime-09 - Previewing a release SDK install resolves the newest stable ROCm 10 release for this GPU Given a machine with an AMD GPU When the user dry-runs a release SDK install for this host - Then the SDK preview reports canonical release provenance + Then the SDK preview reports the next-layout release source And the SDK preview requests the device payload for this host's GPU # `rollback` can only ever undo one step, not walk a history. A unit test on diff --git a/tests/e2e-cucumber/features/therock_next_generation.feature b/tests/e2e-cucumber/features/therock_next_generation.feature index 77e85917fc..ec5a804139 100644 --- a/tests/e2e-cucumber/features/therock_next_generation.feature +++ b/tests/e2e-cucumber/features/therock_next_generation.feature @@ -3,27 +3,47 @@ Feature: TheRock "next" ROCm 10 install layout # ROCm 10 ships from a different source layout than the canonical multi-arch # release/nightly streams: a separate pip index and a separate tarball catalog # whose listing mixes non-release `-tests-` siblings in with the real dist - # archive. The next layout is an explicit extension, never a default: it is - # reachable only by pinning a ROCm version whose major is 10 or newer AND - # naming an exact raw GFX arch, and the canonical streams keep resolving - # exactly as they do today when neither is asked for. + # archive. An unpinned release install reads that layout and takes the newest + # stable version there. It still needs an exact raw GFX arch. Nightly + # auto-select, and any pin that is not a stable ROCm 10 version, stay on the + # canonical streams. # # Scenarios 01-05 point source overrides at loopback fixtures through the # explicit trust gate. Scenario 06 proves the same overrides are inert when # the trust gate is absent; scenario 07 supplies the live GPU evidence. - # The regression this pins: a "next-first" dispatch that probes the ROCm 10 - # sources before the canonical ones. An unpinned release install must never - # look at the next layout at all, so the next fixture is served and asserted - # untouched rather than left unconfigured. - @id:therock-next-01-unpinned-release-stays-canonical - Scenario: therock-next-01 - An unpinned release install stays on the canonical multi-arch source - Given a canonical release pip index fixture and a ROCm 10 pip index fixture + # An unpinned release install resolves the newest stable version on the next + # layout. The fixture publishes 10.0.0, 10.1.0, and a newer prerelease on the + # next index, and 7.14.1 on the canonical index. Falling back to canonical, or + # taking the prerelease, fails the assertions below. + @id:therock-next-01-unpinned-release-selects-newest-stable + Scenario: therock-next-01 - An unpinned release install selects the newest stable ROCm 10 release + Given a canonical release pip index and a ROCm 10 index publishing 10.0.0, 10.1.0, and a newer prerelease When the user previews a wheel SDK install for arch gfx1200 with no version pin + Then the preview selects ROCm 10.1.0 from the ROCm 10 pip index + And the preview reports the next source layout generation + And the preview does not select the canonical release or the newer prerelease + + # A stable pin below ROCm 10 stays on the canonical fixture. The version is + # the one that fixture publishes (7.10.0); 7.14.1 is the same decision and is + # covered by `requested_source_layout_reads_the_request_not_the_resolution`. + @id:therock-next-16-older-pin-stays-canonical + Scenario: therock-next-16 - A ROCm 7 pin stays on the canonical multi-arch source + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 7.10.0 Then the preview resolves the canonical release pip index And the preview reports the canonical multi-arch source layout generation And the preview never mentions the ROCm 10 pip index + # No silent fallback when the family is a group label. Reverting the default + # route to the canonical stream would make this preview succeed. + @id:therock-next-15-unpinned-release-group-family-refused + Scenario: therock-next-15 - An unpinned release wheel install refuses a group family label + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user previews a wheel SDK install for family gfx120X-all with no version pin + Then the install fails + And the failure asks for an exact GPU arch and names --family gfx1200 + # `--version 10.0.0` plus a raw arch is the whole opt-in. The device payload is # requested as an exact `device-gfx1200` extra on rocm, torch and torchvision; # torchaudio carries no device extra because it publishes none. @@ -148,7 +168,8 @@ Feature: TheRock "next" ROCm 10 install layout # # The serve+inference tail matters because `serve-vllm-inference` # (model_serving.feature) does not cover this route: its runtime comes from - # a plain unpinned `rocm install sdk`, never the ROCm 10 preview source. A + # a plain unpinned `rocm install sdk`, which now uses this same ROCm 10 + # index, but that scenario does not assert the vLLM discovery pins. A # discovered wheel that installs cleanly but can't actually load or answer a # request (bad torch realignment, cp314 ABI mismatch, a `uv.toml` fallback # pulling an incompatible transitive dep) would pass every assertion above diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index c7e6197d42..48d847b098 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -58,9 +58,21 @@ async fn user_previews_sdk_install(world: &mut E2eWorld) { // `--family` because a host with no AMD GPU has no target to detect, and the // preview resolves the install folder before it needs one. `--dry-run` keeps // this to a plan: no venv, no multi-GiB download. + // `--channel nightly` because an unpinned release install now reads the + // ROCm 10 layout and refuses a group family. This scenario only needs a + // successful plan so it can read the folder the plan names; nightly + // auto-select still accepts `gfx110X-all`. let stdout = crate::run_rocm_ok( world, - &["install", "sdk", "--family", "gfx110X-all", "--dry-run"], + &[ + "install", + "sdk", + "--channel", + "nightly", + "--family", + "gfx110X-all", + "--dry-run", + ], ); world.cli_output = Some(stdout); } @@ -277,7 +289,19 @@ async fn user_installs_other_family_without_yes(world: &mut E2eWorld) { // `active_default_runtime_relation` shared by both formats — with a family // the host has genuinely never held. The refusal still costs only the // catalog listing: it bails before the multi-GiB archive is fetched. - let args = ["install", "sdk", "--format", "tarball", "--family", family]; + // `--channel nightly` so a group family still reaches the consent gate. + // An unpinned release tarball now requires an exact arch and would stop + // before that gate. + let args = [ + "install", + "sdk", + "--channel", + "nightly", + "--format", + "tarball", + "--family", + family, + ]; let (stdout, stderr, rc) = crate::run_rocm(world, &args); world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); @@ -758,9 +782,9 @@ async fn user_dry_runs_release_sdk(world: &mut E2eWorld) { world.cli_output = Some(stdout); } -#[then("the SDK preview reports canonical release provenance")] -async fn assert_canonical_release_provenance(world: &mut E2eWorld) { - const AGGREGATE: &str = "https://repo.amd.com/rocm/whl-multi-arch"; +#[then("the SDK preview reports the next-layout release source")] +async fn assert_next_release_provenance(world: &mut E2eWorld) { + const NEXT_INDEX: &str = "https://stable.repo.amd.com/rocm/whl-next"; let output = world.cli_output.as_deref().expect("no SDK preview output"); assert_eq!( @@ -768,24 +792,22 @@ async fn assert_canonical_release_provenance(world: &mut E2eWorld) { "release", "the preview is not previewing the release channel:\n{output}" ); - // rocm-cli#271 was this URL with `/{family}` glued on the end: a per-family - // index frozen at 7.13.0 that publishes no device payloads at all. The - // release channel now reads one flat aggregate, and `index_url` is the line - // that says so — the URL the resolver chose, not the one it advertises. + // An unpinned release install reads the ROCm 10 index, not the canonical + // multi-arch index. `index_url` is the URL the resolver chose. assert_eq!( preview_field(output, "canonical_source"), - AGGREGATE, - "the preview declares a source other than the canonical release aggregate:\n{output}" + NEXT_INDEX, + "the preview declares a source other than the ROCm 10 index:\n{output}" ); assert_eq!( preview_field(output, "index_url"), - AGGREGATE, - "the release install resolved something other than the flat aggregate index:\n{output}" + NEXT_INDEX, + "the release install resolved something other than the ROCm 10 index:\n{output}" ); assert_eq!( preview_field(output, "source_layout_generation"), - "multi-arch-v2", - "the preview read the aggregate as some other layout:\n{output}" + "next-v1", + "the preview read the ROCm 10 index as some other layout:\n{output}" ); // The release channel installs stable versions only, and a stable version // encodes no build date. Reporting that instead of a date is the honest diff --git a/tests/e2e-cucumber/tests/e2e/therock_steps.rs b/tests/e2e-cucumber/tests/e2e/therock_steps.rs index 102627612a..76819b021f 100644 --- a/tests/e2e-cucumber/tests/e2e/therock_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/therock_steps.rs @@ -8,9 +8,9 @@ //! ("next") layout from one loopback server, under `current/` and `next/` //! prefixes, and point the gated `ROCM_CLI_THEROCK_*_BASE` overrides at them. //! Serving both — rather than only the one a scenario expects to be used — is -//! what makes "the canonical stream is still canonical" and "the next stream is -//! only reached when explicitly pinned" assertable: a dispatch regression -//! resolves the *other* fixture instead of failing to resolve anything. +//! what makes "the canonical stream is still canonical" and "an unpinned release +//! install reaches the next stream" assertable: a dispatch regression resolves +//! the *other* fixture instead of failing to resolve anything. //! //! Scenario 06 sets an override without the trust opt-in and resolves the live //! default index, proving the child CLI ignores the untrusted value end to end. @@ -269,7 +269,7 @@ fn serve_pip_fixtures(world: &mut E2eWorld) { /// — but torchaudio requires neither, so nothing in its metadata ties it to a /// build. `torchaudio` lists the torchaudio builds the index publishes, each /// with its wheel tag. -fn serve_two_next_builds(world: &mut E2eWorld, torchaudio: &[(&str, &str)]) { +fn write_two_next_builds(world: &E2eWorld, torchaudio: &[(&str, &str)]) { let served = root(world).join("therock-pip-fixtures").join("next"); write_fixture(&served.join("index.html"), &aggregate_root_html()); let builds = [ @@ -298,6 +298,47 @@ fn serve_two_next_builds(world: &mut E2eWorld, torchaudio: &[(&str, &str)]) { write_wheel_index_builds(&served.join("torch"), "torch", &torch); write_wheel_index_builds(&served.join("torchvision"), "torchvision", &torchvision); write_wheel_index_builds(&served.join("torchaudio"), "torchaudio", &torchaudio); +} + +fn serve_two_next_builds(world: &mut E2eWorld, torchaudio: &[(&str, &str)]) { + write_two_next_builds(world, torchaudio); + serve_pip_fixtures(world); +} + +const NEWER_PRERELEASE_ROCM_VERSION: &str = "10.2.0rc1"; +const CANONICAL_STABLE_ROCM_VERSION: &str = "7.14.1"; + +/// Canonical 7.14.1 beside a next index that publishes 10.0.0, 10.1.0, and a +/// prerelease newer than both. An unpinned release preview that falls back to +/// the canonical index, or that takes the prerelease, is visible in the plan. +#[given( + "a canonical release pip index and a ROCm 10 index publishing 10.0.0, 10.1.0, and a newer prerelease" +)] +async fn pip_indexes_for_unpinned_newest_stable(world: &mut E2eWorld) { + let served = root(world).join("therock-pip-fixtures"); + write_pip_index( + &served.join("current"), + [ + ("rocm", CANONICAL_STABLE_ROCM_VERSION), + ("torch", "2.11.0+rocm7.14.1"), + ("torchvision", "0.26.0+rocm7.14.1"), + ("torchaudio", "2.11.0+rocm7.14.1"), + ], + ); + write_two_next_builds( + world, + &[ + (NEXT_TORCHAUDIO_VERSION, ANY_WHEEL_TAG), + (NEWER_NEXT_TORCHAUDIO_VERIFIED_VERSION, ANY_WHEEL_TAG), + ], + ); + let rocm_index = served.join("next").join("rocm").join("index.html"); + let mut html = std::fs::read_to_string(&rocm_index).expect("rocm index was just written"); + let prerelease = format!("rocm-{NEWER_PRERELEASE_ROCM_VERSION}-{ANY_WHEEL_TAG}.whl"); + html.push_str(&format!( + "{prerelease}\n" + )); + std::fs::write(&rocm_index, html).expect("failed to add the prerelease link"); serve_pip_fixtures(world); } @@ -602,19 +643,24 @@ async fn paced_tarball_fixture(world: &mut E2eWorld) { .as_ref() .expect("paced download server was just started") .base_url(); + let catalog = format!("{base}/tarball/current/"); world.command_env.push(( "ROCM_CLI_THEROCK_RELEASE_TARBALL_BASE", - format!("{base}/tarball/current/").into(), + catalog.clone().into(), )); + // An unpinned release tarball reads the next catalog. Point that override + // at the same paced files so the download under test is still this fixture. + world + .command_env + .push(("ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", catalog.into())); } #[when("the user installs the tarball SDK for family gfx120X-all under a real terminal")] async fn install_tarball_sdk_under_pty(world: &mut E2eWorld) { - // No `--version`: tarball installs only accept an explicit version pin for - // a stable ROCm 10+ selector (see `resolve_tarball_artifact_with_timeout`'s - // rejection message), so an unpinned request is what reaches the canonical - // release catalog here — which the given-step populated with exactly one - // candidate, `CURRENT_TARBALL`. + // No `--version`: an unpinned release tarball reads the next catalog. The + // given-step points that catalog at the paced fixture, which publishes + // exactly one stable archive, `CURRENT_TARBALL`. `--family gfx1200` is an + // exact arch; a group label is refused on this route. // // This scenario is testing the download spinner, not the post-install // torch-runtime-dependency setup — but a real SDK install completing @@ -636,7 +682,7 @@ async fn install_tarball_sdk_under_pty(world: &mut E2eWorld) { "--format", "tarball", "--family", - GROUP_FAMILY, + RAW_ARCH, "--yes", ], ) @@ -781,6 +827,44 @@ async fn preview_unpinned_wheel_install(world: &mut E2eWorld) { ); } +#[when("the user previews a wheel SDK install for family gfx120X-all with no version pin")] +async fn preview_unpinned_wheel_install_with_group_family(world: &mut E2eWorld) { + preview( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + GROUP_FAMILY, + "--dry-run", + ], + ); +} + +#[when("the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 7.10.0")] +async fn preview_older_pinned_wheel_install(world: &mut E2eWorld) { + preview_ok( + world, + &[ + "install", + "sdk", + "--channel", + "release", + "--format", + "wheel", + "--family", + RAW_ARCH, + "--version", + CURRENT_ROCM_VERSION, + "--dry-run", + ], + ); +} + #[when("the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0")] async fn preview_pinned_wheel_install(world: &mut E2eWorld) { preview_ok( @@ -1010,16 +1094,56 @@ async fn preview_never_mentions_next_index(world: &mut E2eWorld) { let next = next_pip_base(world); assert!( !stdout(world).contains(&next), - "an unpinned release install reached the ROCm 10 index {next}:\n{}", + "this install reached the ROCm 10 index {next}:\n{}", stdout(world) ); assert!( !stdout(world).contains("next-v1"), - "an unpinned release install selected the next source layout:\n{}", + "this install selected the next source layout:\n{}", stdout(world) ); } +#[then("the preview selects ROCm 10.1.0 from the ROCm 10 pip index")] +async fn preview_selects_newest_stable_next(world: &mut E2eWorld) { + let base = next_pip_base(world); + assert_contains(world, &format!("canonical_source: {base}"), "next source"); + assert_contains(world, &format!("index_url: {base}"), "resolved index"); + assert_contains( + world, + &format!("selected_rocm_version: {NEWER_NEXT_ROCM_VERSION}"), + "selected version", + ); + assert_contains( + world, + &format!("latest_compatible_version: {NEWER_NEXT_ROCM_VERSION}"), + "compatible version", + ); + assert_contains(world, "python_wheel_tag: cp314", "next interpreter"); +} + +#[then("the preview does not select the canonical release or the newer prerelease")] +async fn preview_skips_canonical_and_prerelease(world: &mut E2eWorld) { + let output = stdout(world); + let current = current_pip_base(world); + assert!( + !output.contains(¤t), + "an unpinned release install reached the canonical index {current}:\n{output}" + ); + assert!( + !output.contains(CANONICAL_STABLE_ROCM_VERSION), + "an unpinned release install selected canonical {CANONICAL_STABLE_ROCM_VERSION}:\n{output}" + ); + assert!( + !output.contains(NEWER_PRERELEASE_ROCM_VERSION), + "an unpinned release install selected the prerelease:\n{output}" + ); + assert!( + !output.contains(NEXT_ROCM_VERSION), + "an unpinned release install selected the older stable {NEXT_ROCM_VERSION}:\n{output}" + ); +} + #[then("the preview resolves the ROCm 10 pip index")] async fn preview_resolves_next_pip_index(world: &mut E2eWorld) { let base = next_pip_base(world); diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 1b2280f9dc..a9e682bccc 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -4112,8 +4112,8 @@ fi for workflow_name in ["e2e-selfhosted.yml", "nightly.yml"] { let workflow = read_workflow(workflow_name); assert!( - workflow.contains("e2e-prewarm-multi-arch-v2"), - "{workflow_name} must isolate the canonical multi-arch runtime tree" + workflow.contains("e2e-prewarm-next-v1"), + "{workflow_name} must isolate the ROCm 10 runtime tree from the previous layout" ); assert!( !workflow.lines().any(|line| { From cd38438518983738dc66914d7c807c64d617b051 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 10:28:06 +0200 Subject: [PATCH 2/2] fix(sdk): address review of the unpinned ROCm 10 default - Fix clippy::format_push_string in the newest-stable fixture step (writeln!). - Move the update-skips-layout-preflight guard into preflight_fresh_install_layout and test it: an update with a recorded canonical layout on a grouped-family host passes, a fresh unpinned release install refuses. - Test version-first release ordering on the canonical tarball catalog, and that nightly still orders by upload time. - Drop the `--version 10.1` -> Canonical assertion. Two-component pins are handled on a separate branch, and this PR should not pin that behaviour. - Move therock-next-15/16 after -14 so scenario indexes stay sequential in declaration order (feature_naming test). - PTY scenario steps now say "arch gfx1200" and "a paced release tarball fixture", matching what they run. Scope runtime-08's "both channels" comment to the nightly channel. Signed-off-by: Roman Inflianskas --- apps/rocm/src/therock.rs | 105 +++++++++++++++--- .../features/download_progress_pty.feature | 4 +- .../features/runtime_setup.feature | 5 +- .../features/therock_next_generation.feature | 47 ++++---- tests/e2e-cucumber/tests/e2e/therock_steps.rs | 12 +- 5 files changed, 125 insertions(+), 48 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 6009b2db6f..4b08f8454f 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -1458,6 +1458,25 @@ fn save_startup_update_check(paths: &AppPaths, record: &StartupUpdateCheckRecord ) } +/// The fail-fast layout check [`install_wheel_runtime`] runs before it +/// provisions Python. A fresh install (`layout_override` is `None`) asks +/// [`select_source_layout`], so a request it could only refuse stops here. An +/// update passes the layout its manifest recorded and skips the check: +/// re-deciding from the update's absent selector would treat a canonical +/// runtime's refresh as an unpinned release install, and refuse it on a host +/// that names only a grouped family. +fn preflight_fresh_install_layout( + layout_override: Option, + channel: TheRockChannel, + version_selector: Option<&RuntimeVersionSelector>, + family_resolution: impl FnOnce() -> Result, +) -> Result<()> { + if layout_override.is_some() { + return Ok(()); + } + select_source_layout(channel, &family_resolution()?, version_selector).map(|_| ()) +} + #[allow(clippy::too_many_arguments)] fn install_wheel_runtime( paths: &AppPaths, @@ -1484,16 +1503,10 @@ fn install_wheel_runtime( // Fail fast on a fresh request `resolve_pip_runtime` could only ever reject // (wrong channel, or a grouped family with no exact arch) before paying for // Python provisioning below, which can mean a real download for a layout - // this host or channel can never actually use. An update already recorded - // its layout; re-deciding from the absent selector would treat a canonical - // runtime's refresh as an unpinned release install. - if layout_override.is_none() { - select_source_layout( - channel, - &resolve_family(paths, recovered_family_override.as_deref())?, - version_selector, - )?; - } + // this host or channel can never actually use. + preflight_fresh_install_layout(layout_override, channel, version_selector, || { + resolve_family(paths, recovered_family_override.as_deref()) + })?; // The interpreter has to be picked before `resolve_pip_runtime`, which // selects wheels using that interpreter's own tags. `requested_source_layout` // answers from the *request*: an unpinned release install and an explicit @@ -7442,12 +7455,6 @@ mod tests { .version, "3.12" ); - // A two-component selector does not parse as stable >= 10. - let line_pin = RuntimeVersionSelector::Version("10.1".to_owned()); - assert_eq!( - requested_source_layout(None, TheRockChannel::Release, Some(&line_pin)), - SourceLayout::Canonical - ); assert_eq!( requested_source_layout(None, TheRockChannel::Release, None), @@ -7496,6 +7503,72 @@ mod tests { ); } + /// The canonical release catalog orders by version too: a republished + /// older stable archive with a later mtime does not beat a newer stable. + /// The nightly catalog keeps upload time as its first key. + #[test] + fn canonical_tarball_release_orders_by_version_and_nightly_by_upload_time() { + let platform = platform_tarball_token(); + let prefix = format!("therock-dist-{platform}-gfx110X-all-"); + let files = vec![ + TarballIndexFile { + name: format!("{prefix}7.14.0.tar.gz"), + mtime: 30.0, + }, + TarballIndexFile { + name: format!("{prefix}7.14.1.tar.gz"), + mtime: 10.0, + }, + ]; + + let release = select_tarball_candidate( + &files, + TheRockChannel::Release, + SourceLayout::Canonical, + "gfx110X-all", + None, + ) + .map(|(_, version)| version); + assert_eq!(release.as_deref(), Some("7.14.1")); + + let nightly = select_tarball_candidate( + &files, + TheRockChannel::Nightly, + SourceLayout::Canonical, + "gfx110X-all", + None, + ) + .map(|(_, version)| version); + assert_eq!( + nightly.as_deref(), + Some("7.14.0"), + "the nightly catalog picks the latest upload" + ); + } + + /// An update passes the layout its manifest recorded. On a host that names + /// only a grouped family, the preflight must let that update through + /// instead of judging it as an unpinned release install, which would need + /// an exact arch. A fresh unpinned release install on the same host still + /// refuses. + #[test] + fn preflight_skips_the_layout_check_for_an_update_with_a_recorded_layout() { + let grouped = || Ok(family_for_test("gfx110X-all", None)); + + preflight_fresh_install_layout( + Some(SourceLayout::Canonical), + TheRockChannel::Release, + None, + grouped, + ) + .expect("updating a canonical runtime must not demand an exact arch"); + + let error = preflight_fresh_install_layout(None, TheRockChannel::Release, None, grouped) + .expect_err("a fresh unpinned release install needs an exact arch") + .to_string(); + assert!(error.contains("requires an exact GPU arch"), "{error}"); + } + /// Same rule on the tarball catalog: a later mtime on an older stable, or /// on a prerelease, does not beat the newest stable version. #[test] diff --git a/tests/e2e-cucumber/features/download_progress_pty.feature b/tests/e2e-cucumber/features/download_progress_pty.feature index bab70dcc35..247826e216 100644 --- a/tests/e2e-cucumber/features/download_progress_pty.feature +++ b/tests/e2e-cucumber/features/download_progress_pty.feature @@ -17,8 +17,8 @@ Feature: Download-progress spinner under a real terminal # this scenario alone. Serial execution removes that contention. @id:download-progress-linux-tarball-install-shows-live-progress @requires-os:linux @serial Scenario: download-progress-01 - The tarball download spinner renders progress and clears on completion - Given a paced canonical release tarball fixture - When the user installs the tarball SDK for family gfx120X-all under a real terminal + Given a paced release tarball fixture + When the user installs the tarball SDK for arch gfx1200 under a real terminal Then the terminal shows an intermediate download progress frame And the terminal shows the archive being extracted And the tarball install exits cleanly diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 2ceb53333e..b38da3a0fc 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -144,9 +144,10 @@ Feature: Runtime configuration Then the adoption is refused And the error explains which install types can be adopted - # Both channels now resolve from one canonical aggregate index each, so the + # The nightly channel resolves from one canonical aggregate index, and an + # unpinned release install from the ROCm 10 index (runtime-09 below), so the # provenance the preview prints is the whole of what the user can check before - # committing to a multi-GiB install. This pins the nightly half: the source that + # committing to a multi-GiB install. This pins the nightly case: the source that # was selected, the version that came back from it, and the layout generation # that source was read as. `--family` is supplied so the scenario needs no GPU, # and `@nightly` because it resolves the real index over the network — the same diff --git a/tests/e2e-cucumber/features/therock_next_generation.feature b/tests/e2e-cucumber/features/therock_next_generation.feature index ec5a804139..6d5fe5a449 100644 --- a/tests/e2e-cucumber/features/therock_next_generation.feature +++ b/tests/e2e-cucumber/features/therock_next_generation.feature @@ -8,9 +8,10 @@ Feature: TheRock "next" ROCm 10 install layout # auto-select, and any pin that is not a stable ROCm 10 version, stay on the # canonical streams. # - # Scenarios 01-05 point source overrides at loopback fixtures through the - # explicit trust gate. Scenario 06 proves the same overrides are inert when - # the trust gate is absent; scenario 07 supplies the live GPU evidence. + # Scenarios 01-05, 15 and 16 point source overrides at loopback fixtures + # through the explicit trust gate. Scenario 06 proves the same overrides are + # inert when the trust gate is absent; scenario 07 supplies the live GPU + # evidence. # An unpinned release install resolves the newest stable version on the next # layout. The fixture publishes 10.0.0, 10.1.0, and a newer prerelease on the @@ -24,26 +25,6 @@ Feature: TheRock "next" ROCm 10 install layout And the preview reports the next source layout generation And the preview does not select the canonical release or the newer prerelease - # A stable pin below ROCm 10 stays on the canonical fixture. The version is - # the one that fixture publishes (7.10.0); 7.14.1 is the same decision and is - # covered by `requested_source_layout_reads_the_request_not_the_resolution`. - @id:therock-next-16-older-pin-stays-canonical - Scenario: therock-next-16 - A ROCm 7 pin stays on the canonical multi-arch source - Given a canonical release pip index fixture and a ROCm 10 pip index fixture - When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 7.10.0 - Then the preview resolves the canonical release pip index - And the preview reports the canonical multi-arch source layout generation - And the preview never mentions the ROCm 10 pip index - - # No silent fallback when the family is a group label. Reverting the default - # route to the canonical stream would make this preview succeed. - @id:therock-next-15-unpinned-release-group-family-refused - Scenario: therock-next-15 - An unpinned release wheel install refuses a group family label - Given a canonical release pip index fixture and a ROCm 10 pip index fixture - When the user previews a wheel SDK install for family gfx120X-all with no version pin - Then the install fails - And the failure asks for an exact GPU arch and names --family gfx1200 - # `--version 10.0.0` plus a raw arch is the whole opt-in. The device payload is # requested as an exact `device-gfx1200` extra on rocm, torch and torchvision; # torchaudio carries no device extra because it publishes none. @@ -272,3 +253,23 @@ Feature: TheRock "next" ROCm 10 install layout When the user installs a wheel SDK for arch gfx1200 pinned to ROCm 10.0.0 without confirming Then the install is refused without replacing the active default runtime And the install log notes that its torchaudio is the newest build for ROCm 10.0.0 and not hardware-verified + + # No silent fallback when the family is a group label. Reverting the default + # route to the canonical stream would make this preview succeed. + @id:therock-next-15-unpinned-release-group-family-refused + Scenario: therock-next-15 - An unpinned release wheel install refuses a group family label + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user previews a wheel SDK install for family gfx120X-all with no version pin + Then the install fails + And the failure asks for an exact GPU arch and names --family gfx1200 + + # A stable pin below ROCm 10 stays on the canonical fixture. The version is + # the one that fixture publishes (7.10.0); 7.14.1 is the same decision and is + # covered by `requested_source_layout_reads_the_request_not_the_resolution`. + @id:therock-next-16-older-pin-stays-canonical + Scenario: therock-next-16 - A ROCm 7 pin stays on the canonical multi-arch source + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 7.10.0 + Then the preview resolves the canonical release pip index + And the preview reports the canonical multi-arch source layout generation + And the preview never mentions the ROCm 10 pip index diff --git a/tests/e2e-cucumber/tests/e2e/therock_steps.rs b/tests/e2e-cucumber/tests/e2e/therock_steps.rs index 76819b021f..168081cf09 100644 --- a/tests/e2e-cucumber/tests/e2e/therock_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/therock_steps.rs @@ -335,9 +335,11 @@ async fn pip_indexes_for_unpinned_newest_stable(world: &mut E2eWorld) { let rocm_index = served.join("next").join("rocm").join("index.html"); let mut html = std::fs::read_to_string(&rocm_index).expect("rocm index was just written"); let prerelease = format!("rocm-{NEWER_PRERELEASE_ROCM_VERSION}-{ANY_WHEEL_TAG}.whl"); - html.push_str(&format!( - "{prerelease}\n" - )); + writeln!( + html, + "{prerelease}" + ) + .expect("writing to a String cannot fail"); std::fs::write(&rocm_index, html).expect("failed to add the prerelease link"); serve_pip_fixtures(world); } @@ -605,7 +607,7 @@ const PACED_TARBALL_CHUNK_DELAY: Duration = Duration::from_millis(150); /// `engines_steps.rs`'s file-local `SCREEN_TIMEOUT` convention. const PTY_SCREEN_TIMEOUT: Duration = Duration::from_secs(30); -#[given("a paced canonical release tarball fixture")] +#[given("a paced release tarball fixture")] async fn paced_tarball_fixture(world: &mut E2eWorld) { let served = root(world).join("therock-paced-tarball-fixture"); write_fixture( @@ -655,7 +657,7 @@ async fn paced_tarball_fixture(world: &mut E2eWorld) { .push(("ROCM_CLI_THEROCK_NEXT_TARBALL_BASE", catalog.into())); } -#[when("the user installs the tarball SDK for family gfx120X-all under a real terminal")] +#[when("the user installs the tarball SDK for arch gfx1200 under a real terminal")] async fn install_tarball_sdk_under_pty(world: &mut E2eWorld) { // No `--version`: an unpinned release tarball reads the next catalog. The // given-step points that catalog at the paced fixture, which publishes