From e74afaf48501a0ab7ad9bc8bad0bb00d857007b6 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:08:45 +0000 Subject: [PATCH 01/11] feat(core): look for an installed uv without acquiring one ensure_uv_binary returns the binary named by ROCM_CLI_UV_BINARY or the managed copy, and otherwise takes a lock under the data folder's tools folder and downloads one. A caller that must leave the machine untouched (an install preview) had no way to ask whether a uv is already there. find_uv_binary answers that question and nothing more: the override, or the managed copy when it runs, and None when ensure_uv_binary would have had to fetch. It creates no folder and takes no lock. The managed copy's location now comes from one helper shared with the install, and a test shows the lookup finds the copy the install reuses. No caller yet; ensure_uv_binary behaves exactly as before. Signed-off-by: Jussi Elo --- crates/rocm-core/src/lib.rs | 7 ++- crates/rocm-core/src/uv.rs | 106 +++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c142c3332f..8a9de16e69 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -222,9 +222,10 @@ pub use service_record::{ pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, - UvCacheSource, ViolationSubject, check_dependencies, ensure_uv_binary, split_local_version, - uv_binary_name, uv_cache_source, uv_command_env, uv_http_timeout_secs, uv_pip_check_args, - uv_pip_freeze_args, uv_pip_install_base, uv_venv_args, violation_subject, violations_requiring, + UvCacheSource, ViolationSubject, check_dependencies, ensure_uv_binary, find_uv_binary, + split_local_version, uv_binary_name, uv_cache_source, uv_command_env, uv_http_timeout_secs, + uv_pip_check_args, uv_pip_freeze_args, uv_pip_install_base, uv_venv_args, violation_subject, + violations_requiring, }; pub use vllm_oom::{ VLLM_OOM_DETECTED_PREFIX, VLLM_OOM_DIAGNOSE_COMMAND, vllm_oom_diagnose_symptom, diff --git a/crates/rocm-core/src/uv.rs b/crates/rocm-core/src/uv.rs index 5f3b17b34c..e89a0d1121 100644 --- a/crates/rocm-core/src/uv.rs +++ b/crates/rocm-core/src/uv.rs @@ -422,6 +422,38 @@ pub fn ensure_uv_binary(paths: &AppPaths) -> Result { ) } +/// A `uv` that is already available, without acquiring one: the binary named by +/// [`UV_BINARY_ENV`], or else the managed copy a previous install left behind. +/// +/// This is [`ensure_uv_binary`] stopped short of its download. It never downloads, +/// takes a lock, or creates a folder (not even the managed tools folder), and `None` +/// is the answer when `ensure_uv_binary` would have had to fetch one. A caller that +/// must leave the machine untouched (a preview) asks this instead and goes without +/// `uv` on `None`. A managed copy is only reported once it runs, which is the same +/// test `ensure_uv_binary` applies before it reuses one. +pub fn find_uv_binary(paths: &AppPaths) -> Option { + if let Some(path) = uv_binary_override() { + return Some(path); + } + find_managed_uv(paths, &uv_version()) +} + +/// The installed managed `uv` of `version`, when it runs. Looks and creates nothing. +fn find_managed_uv(paths: &AppPaths, version: &str) -> Option { + usable_uv_in(&managed_uv_install_dir(paths, version), uv_binary_name()) +} + +/// Where the managed `uv` of `version` is installed: `tools/uv/` under the +/// data folder. Shared by the install and by [`find_uv_binary`] so that a copy one +/// installs is the copy the other finds. +fn managed_uv_install_dir(paths: &AppPaths, version: &str) -> PathBuf { + managed_uv_root(paths).join(slug(version)) +} + +fn managed_uv_root(paths: &AppPaths) -> PathBuf { + managed_tools_dir(&paths.data_dir).join("uv") +} + /// Which `uv` release to install, resolved from the environment by /// [`ensure_uv_binary`] so the install itself never reads it. struct ManagedUvRelease { @@ -444,9 +476,9 @@ struct ManagedUvRelease { /// The lock is a leaf: nothing below acquires another lock, so it cannot take part in /// a lock-order inversion with the managed-launch or daemon-autostart locks. fn install_managed_uv(paths: &AppPaths, release: &ManagedUvRelease) -> Result { - let uv_root = managed_tools_dir(&paths.data_dir).join("uv"); + let uv_root = managed_uv_root(paths); let version_slug = slug(&release.version); - let install_dir = uv_root.join(&version_slug); + let install_dir = managed_uv_install_dir(paths, &release.version); let binary_name = uv_binary_name(); if let Some(existing) = usable_uv_in(&install_dir, binary_name) { @@ -1216,6 +1248,76 @@ All installed packages are compatible Ok(std::fs::read(&archive)?) } + /// Plants `script` as the managed `uv` of `version`, where an install would have + /// left it. + #[cfg(unix)] + fn plant_managed_uv(paths: &AppPaths, version: &str, script: &str) -> PathBuf { + use std::os::unix::fs::PermissionsExt; + + let dir = managed_uv_install_dir(paths, version); + std::fs::create_dir_all(&dir).expect("create the managed uv folder"); + let binary = dir.join(uv_binary_name()); + std::fs::write(&binary, script).expect("write the stand-in uv"); + std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o755)) + .expect("make the stand-in uv executable"); + binary + } + + // The preview's question, answered on every platform: on a clean data folder + // there is no managed `uv`, and asking leaves no trace. `ensure_uv_binary` would + // have created `tools/uv` (for its lock) before it downloaded anything, which is + // the state this pins as untouched. + #[test] + fn looking_for_a_managed_uv_on_a_clean_host_finds_none_and_creates_nothing() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-clean"); + assert_eq!(find_managed_uv(&paths, "latest"), None); + assert!( + !managed_tools_dir(&paths.data_dir).exists(), + "looking for uv must not create the managed tools folder" + ); + let _ = std::fs::remove_dir_all(&root); + } + + // Unix-only: the stand-in `uv` is a shell script, and Windows can only run a real + // executable as `uv.exe`. + #[cfg(unix)] + #[test] + fn an_installed_managed_uv_is_found_where_the_install_puts_it() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-installed"); + let planted = plant_managed_uv(&paths, "latest", "#!/bin/sh\nexit 0\n"); + assert_eq!(find_managed_uv(&paths, "latest"), Some(planted.clone())); + // The install reuses the same copy rather than downloading: the lookup and the + // install agree on the location, so a copy one leaves is the copy the other + // finds. The release names a URL nothing listens on, so reaching for it would + // fail the call. + let reused = install_managed_uv( + &paths, + &ManagedUvRelease { + version: "latest".to_owned(), + asset: "uv-test.tar.gz".to_owned(), + url: "http://127.0.0.1:9/uv-test.tar.gz".to_owned(), + }, + ) + .expect("an installed uv is reused without a download"); + assert_eq!(reused, planted); + // Another version is a different copy. + assert_eq!(find_managed_uv(&paths, "0.0.1"), None); + let _ = std::fs::remove_dir_all(&root); + } + + #[cfg(unix)] + #[test] + fn a_managed_uv_that_does_not_run_is_not_reported() { + let (root, paths) = crate::test_support::temp_app_paths("uv-find-broken"); + plant_managed_uv(&paths, "latest", "#!/bin/sh\nexit 1\n"); + assert_eq!( + find_managed_uv(&paths, "latest"), + None, + "a copy `ensure_uv_binary` would replace is not one a preview can use" + ); + let _ = std::fs::remove_dir_all(&root); + } + // Unix-only: the fake `uv` is a shell script, and Windows can only run a real // executable as `uv.exe`. The lock and the re-check under test are // platform-independent. From 19974a4adb6321e8393af444903681b1d80ea2e8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:16:17 +0000 Subject: [PATCH 02/11] refactor(install): write the wheel install plan from one module install_wheel_runtime wrote the sdk install plan inline, from the provenance lines down to the dry-run command and manifest path. Move those writes into therock/wheel_plan.rs behind a WheelPlan value: write_wheel_plan for the lines a preview and a real install share, write_dry_run_tail for the lines only a preview adds. The lines, their order and their wording are unchanged; the existing tests pass as they were. This lets the next change print a plan whose package versions are not chosen yet through the same writer instead of a second copy of it. Signed-off-by: Jussi Elo --- apps/rocm/src/therock.rs | 153 +++++------------------- apps/rocm/src/therock/resolve.rs | 5 +- apps/rocm/src/therock/wheel_plan.rs | 175 ++++++++++++++++++++++++++++ docs/architecture.md | 3 +- 4 files changed, 207 insertions(+), 129 deletions(-) create mode 100644 apps/rocm/src/therock/wheel_plan.rs diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 1cfe879e86..c05b12b1d2 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -22,6 +22,7 @@ mod source; mod test_support; mod update; mod version; +mod wheel_plan; use self::consent::{ SdkInstallApproval, active_default_runtime_relation, confirm_overwrite_existing_sdk, @@ -62,6 +63,7 @@ pub(crate) use self::version::runtime_version_display; use self::version::{ parse_host_version, parse_version_for_ordering, runtime_version_build_date, version_relation, }; +use self::wheel_plan::{WheelPlan, write_dry_run_tail, write_wheel_plan}; #[cfg(test)] pub(crate) use crate::runtime_registry::RocmSdkLibraryProbe; #[cfg(test)] @@ -74,12 +76,13 @@ use crate::runtime_registry::{ ManifestOverwrite, runtime_manifest_path, wheel_composition_includes_devel, write_runtime_manifest, }; +#[cfg(test)] use crate::shell_quote::shell_quote; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, detect_host_gpu_diagnostics, detect_legacy_rocm_summary, ensure_uv_binary, interactive_terminal, runtime_is_windows, runtime_python_executable_in_env, unix_time_millis, - uv_pip_install_base, uv_venv_args, + uv_pip_install_base, }; #[cfg(test)] use rocm_core::{disk_space, managed_uv_cache_dir, normalize_therock_family}; @@ -418,98 +421,10 @@ fn install_wheel_runtime( let install_root = resolved_install_root(paths, "wheel", &runtime_key, prefix); let manifest_path = runtime_manifest_path(paths, &runtime_key); - let mut output = String::new(); - let _ = writeln!(output, "sdk install"); - let _ = writeln!( - output, - " summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system" - ); - render_canonical_provenance( - &mut output, - channel, - &resolution.index_url, - resolution.layout.generation(), - &resolution.latest_version, - ); - let _ = writeln!(output, " format: wheel"); - if let Some(selector) = version_selector { - let _ = writeln!(output, " requested: {}", selector.describe()); - } - let _ = writeln!(output, " target_family: {}", resolution.family); - let _ = writeln!( - output, - " target_family_source: {}", - resolution.family_source - ); - let _ = writeln!(output, " device_target: {}", device_target.as_str()); - if let Some(reason) = device_target.reason() { - let _ = writeln!(output, " device_target_reason: {reason}"); - } - let _ = writeln!(output, " index_url: {}", resolution.index_url); - let _ = writeln!( - output, - " latest_compatible_version: {}", - runtime_version_display(&resolution.latest_version) - ); // Probed once and reused by the progress line below: each call is a full // filesystem scan for a legacy ROCm, and the summary block and the visible // note report the same answer about the same resolved version. let host_version_newer = host_rocm_version_newer_than(&resolution.latest_version); - if let Some(host_version) = host_version_newer.as_deref() { - let _ = writeln!( - output, - " version_note: {}", - wheel_host_version_note( - host_version, - &runtime_version_display(&resolution.latest_version) - ) - ); - } - let _ = writeln!( - output, - " compatibility_key: {}", - runtime_version_display(&resolution.package_versions.compatibility_key) - ); - let _ = writeln!(output, " target: {}", install_root.display()); - let _ = writeln!(output, " runtime_key: {runtime_key}"); - let _ = writeln!(output, " python_launcher: {}", python.launcher_display()); - let _ = writeln!(output, " python_source: {}", python.source().as_str()); - if let Some(note) = python.note() { - let _ = writeln!(output, " python_note: {note}"); - } - let _ = writeln!( - output, - " python_wheel_tag: {}", - wheel_compatibility.python_tag - ); - let _ = writeln!( - output, - " platform_wheel_tags: {}", - wheel_compatibility.platform_tags.join(",") - ); - // What resolution asked for, printed beside what the install will ask for. - // The two are produced by different code paths from the same `include_devel` - // and must name the same extras; showing only the second would hide a - // resolve that constrained the version choice by a toolchain the user - // declined. Absent on the canonical layout, which resolves by scraping each - // package's index instead of compiling a requirement set. - if let Some(specs) = resolution.version_resolution_specs.as_ref() { - let _ = writeln!(output, " version_resolution_specs: {}", specs.join(" ")); - } - // Printed from the same value that chose the pin, so the line cannot claim - // a verified build the requirements above do not name. - if let Some(pin) = resolution.torchaudio_pin.as_ref() { - let _ = writeln!(output, " torchaudio_selection: {}", pin.describe()); - } - let _ = writeln!( - output, - " package_specs: {}", - wheel_composition.package_specs.join(" ") - ); - let _ = writeln!( - output, - " package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction" - ); let no_wheel_warning = repo_version_without_wheels( resolution.newest_repo_version.as_deref(), &resolution.latest_version, @@ -520,44 +435,30 @@ fn install_wheel_runtime( &runtime_version_display(&resolution.latest_version), ) }); - if let Some(warning) = no_wheel_warning.as_deref() { - let _ = writeln!(output, " warning: {warning}"); - } + let plan = WheelPlan { + channel, + version_selector, + index_url: &resolution.index_url, + layout_generation: resolution.layout.generation(), + latest_version: &resolution.latest_version, + family: &resolution.family, + family_source: &resolution.family_source, + device_target: &device_target, + host_version_newer: host_version_newer.as_deref(), + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: &runtime_key, + python: &python, + wheel_compatibility: &wheel_compatibility, + version_resolution_specs: resolution.version_resolution_specs.as_deref(), + torchaudio_pin: resolution.torchaudio_pin.as_ref(), + package_specs: &wheel_composition.package_specs, + no_wheel_warning: no_wheel_warning.as_deref(), + }; + let mut output = String::new(); + write_wheel_plan(&mut output, &plan); if dry_run { - let env_python = venv_python_path(&install_root); - let mut install_args = uv_pip_install_base(&env_python); - install_args.extend(["--index-url".to_owned(), resolution.index_url.clone()]); - if matches!(channel, TheRockChannel::Nightly) { - install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); - } - install_args.extend(wheel_composition.package_specs.iter().cloned()); - let venv_args = uv_venv_args(&python.venv_python_arg(), &install_root); - // This line is offered as the command to run, so it has to be one a - // shell reads back as the argv it previews. The install root in it - // comes from a `--prefix` the local assistant can choose - // (`install_sdk_dry_run` is a model-callable tool), so a bare or - // double-quoted value here would be a model-chosen value in a line a - // user is invited to paste. - let venv_args_display = venv_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); - let install_args_display = install_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); - let _ = writeln!(output, " mode: dry-run"); - let _ = writeln!( - output, - " command: uv {venv_args_display} && uv {install_args_display}" - ); - let _ = writeln!( - output, - " activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process" - ); - let _ = writeln!(output, " manifest: {}", manifest_path.display()); + write_dry_run_tail(&mut output, &plan, &manifest_path); return Ok(SdkInstallResult::plan(output)); } diff --git a/apps/rocm/src/therock/resolve.rs b/apps/rocm/src/therock/resolve.rs index 9efff586db..0dddb3ef51 100644 --- a/apps/rocm/src/therock/resolve.rs +++ b/apps/rocm/src/therock/resolve.rs @@ -14,6 +14,7 @@ pub(super) use self::tarball::{resolve_tarball_artifact, resolve_tarball_artifac #[cfg(test)] pub(super) use self::wheel::select_rocm_version; pub(super) use self::wheel::{ - AggregateDeviceTarget, canonical_auto_select_needs_next_python, resolve_pip_runtime, - resolve_pip_runtime_with_timeout, unverified_torchaudio_note, wheel_runtime_composition, + AggregateDeviceTarget, TorchaudioPin, canonical_auto_select_needs_next_python, + resolve_pip_runtime, resolve_pip_runtime_with_timeout, unverified_torchaudio_note, + wheel_runtime_composition, }; diff --git a/apps/rocm/src/therock/wheel_plan.rs b/apps/rocm/src/therock/wheel_plan.rs new file mode 100644 index 0000000000..6c6cfc125f --- /dev/null +++ b/apps/rocm/src/therock/wheel_plan.rs @@ -0,0 +1,175 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `sdk install` plan a wheel install prints, key by key. +//! +//! A preview prints it and stops; a real install prints it as the head of its +//! own output. [`write_wheel_plan`] writes the lines both share, and +//! [`write_dry_run_tail`] the lines only a preview adds. Moved out of +//! `install_wheel_runtime` so that one writer serves every plan it prints. + +use super::python::{PlannedPython, WheelCompatibility}; +use super::resolve::{AggregateDeviceTarget, TorchaudioPin}; +use super::source::{RuntimeVersionSelector, TheRockChannel}; +use super::{ + render_canonical_provenance, runtime_version_display, venv_python_path, wheel_host_version_note, +}; +use crate::shell_quote::shell_quote; +use rocm_core::{uv_pip_install_base, uv_venv_args}; +use std::fmt::Write as _; +use std::path::Path; + +/// The facts the plan prints, from a resolution and the Python decision. +pub(super) struct WheelPlan<'a> { + pub(super) channel: TheRockChannel, + pub(super) version_selector: Option<&'a RuntimeVersionSelector>, + pub(super) index_url: &'a str, + pub(super) layout_generation: &'a str, + /// The `rocm` version the plan is for. + pub(super) latest_version: &'a str, + pub(super) family: &'a str, + pub(super) family_source: &'a str, + pub(super) device_target: &'a AggregateDeviceTarget, + /// The host's ROCm when it is newer than `latest_version`. + pub(super) host_version_newer: Option<&'a str>, + pub(super) compatibility_key: &'a str, + pub(super) install_root: &'a Path, + pub(super) runtime_key: &'a str, + pub(super) python: &'a PlannedPython, + pub(super) wheel_compatibility: &'a WheelCompatibility, + /// The requirement lines version resolution asked `uv` about. `None` on the + /// canonical layout, which resolves without `uv`. + pub(super) version_resolution_specs: Option<&'a [String]>, + pub(super) torchaudio_pin: Option<&'a TorchaudioPin>, + pub(super) package_specs: &'a [String], + pub(super) no_wheel_warning: Option<&'a str>, +} + +/// Writes the plan from `sdk install` to `package_policy` and the repository +/// warning. A real install carries on from here; a preview adds +/// [`write_dry_run_tail`]. +pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { + let _ = writeln!(output, "sdk install"); + let _ = writeln!( + output, + " summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system" + ); + render_canonical_provenance( + output, + plan.channel, + plan.index_url, + plan.layout_generation, + plan.latest_version, + ); + let _ = writeln!(output, " format: wheel"); + if let Some(selector) = plan.version_selector { + let _ = writeln!(output, " requested: {}", selector.describe()); + } + let _ = writeln!(output, " target_family: {}", plan.family); + let _ = writeln!(output, " target_family_source: {}", plan.family_source); + let _ = writeln!(output, " device_target: {}", plan.device_target.as_str()); + if let Some(reason) = plan.device_target.reason() { + let _ = writeln!(output, " device_target_reason: {reason}"); + } + let _ = writeln!(output, " index_url: {}", plan.index_url); + let _ = writeln!( + output, + " latest_compatible_version: {}", + runtime_version_display(plan.latest_version) + ); + if let Some(host_version) = plan.host_version_newer { + let _ = writeln!( + output, + " version_note: {}", + wheel_host_version_note(host_version, &runtime_version_display(plan.latest_version)) + ); + } + let _ = writeln!( + output, + " compatibility_key: {}", + runtime_version_display(plan.compatibility_key) + ); + let _ = writeln!(output, " target: {}", plan.install_root.display()); + let _ = writeln!(output, " runtime_key: {}", plan.runtime_key); + let _ = writeln!( + output, + " python_launcher: {}", + plan.python.launcher_display() + ); + let _ = writeln!(output, " python_source: {}", plan.python.source().as_str()); + if let Some(note) = plan.python.note() { + let _ = writeln!(output, " python_note: {note}"); + } + let _ = writeln!( + output, + " python_wheel_tag: {}", + plan.wheel_compatibility.python_tag + ); + let _ = writeln!( + output, + " platform_wheel_tags: {}", + plan.wheel_compatibility.platform_tags.join(",") + ); + // What resolution asked for, printed beside what the install will ask for. + // The two are produced by different code paths from the same `include_devel` + // and must name the same extras; showing only the second would hide a + // resolve that constrained the version choice by a toolchain the user + // declined. Absent on the canonical layout, which resolves by scraping each + // package's index instead of compiling a requirement set. + if let Some(specs) = plan.version_resolution_specs { + let _ = writeln!(output, " version_resolution_specs: {}", specs.join(" ")); + } + // Printed from the same value that chose the pin, so the line cannot claim + // a verified build the requirements above do not name. + if let Some(pin) = plan.torchaudio_pin { + let _ = writeln!(output, " torchaudio_selection: {}", pin.describe()); + } + let _ = writeln!(output, " package_specs: {}", plan.package_specs.join(" ")); + let _ = writeln!( + output, + " package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction" + ); + if let Some(warning) = plan.no_wheel_warning { + let _ = writeln!(output, " warning: {warning}"); + } +} + +/// The lines a preview adds after [`write_wheel_plan`]: the commands a real install +/// would run, and where it would record the runtime. +pub(super) fn write_dry_run_tail(output: &mut String, plan: &WheelPlan<'_>, manifest_path: &Path) { + let env_python = venv_python_path(plan.install_root); + let mut install_args = uv_pip_install_base(&env_python); + install_args.extend(["--index-url".to_owned(), plan.index_url.to_owned()]); + if matches!(plan.channel, TheRockChannel::Nightly) { + install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); + } + install_args.extend(plan.package_specs.iter().cloned()); + let venv_args = uv_venv_args(&plan.python.venv_python_arg(), plan.install_root); + // This line is offered as the command to run, so it has to be one a + // shell reads back as the argv it previews. The install root in it + // comes from a `--prefix` the local assistant can choose + // (`install_sdk_dry_run` is a model-callable tool), so a bare or + // double-quoted value here would be a model-chosen value in a line a + // user is invited to paste. + let venv_args_display = venv_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let install_args_display = install_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let _ = writeln!(output, " mode: dry-run"); + let _ = writeln!( + output, + " command: uv {venv_args_display} && uv {install_args_display}" + ); + let _ = writeln!( + output, + " activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process" + ); + let _ = writeln!(output, " manifest: {}", manifest_path.display()); +} diff --git a/docs/architecture.md b/docs/architecture.md index 9508db8415..0fa6e056ec 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -100,7 +100,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and reads the os-release every one of these plans is built from through `rocm_core::hardware_root::real_os_release`, which has no copy here. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `rocm_core::hardware_root::real_os_release` and the three OpenMPI/libatomic/libnuma install hints ignore a simulated host root, and a scan that no source file of the binary reads os-release by hand. -- [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). +- [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the plan itself being written by `therock/wheel_plan.rs`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. - [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. - [`therock/process.rs`](../apps/rocm/src/therock/process.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for scratch directories (split out of `therock.rs`): running and capturing child commands (`run_command*`, `capture_command_output*`, `capture_python_stdout*`, `run_uv_progress_command`, and the Windows temp-file variants with `windows_child_path`), the bounded wait (`wait_with_output_bounded`), the progress-line output with its `--json` suppression guard (`SuppressProgressOutput`, `progress_line`), and unpacking the SDK tarball (`extract_tarball`, `extract_tarball_and_discard_archive`, which is why it sits here and not in `io.rs`: it spawns `tar` through `run_command`). Its `archive_properties` property tests run on Unix only. The process-wide environment lock these tests take is the single `PROCESS_ENV_TEST_LOCK` in [`test_support.rs`](../apps/rocm/src/test_support.rs), shared with the tests left in `therock.rs`. @@ -113,6 +113,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. - [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. +- [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) — mechanical relocation out of `install_wheel_runtime` in `therock.rs`, a leaf child of `therock.rs` that builds on `therock/python.rs` and `therock/resolve.rs` for the Python decision and the resolution it prints: the `sdk install` plan a wheel install prints, key by key. `WheelPlan` carries the facts, `write_wheel_plan` writes the lines a preview and a real install share, and `write_dry_run_tail` the lines only a preview adds (the `uv` commands it would run and the manifest path). One writer serves every plan, so a line the plan gains appears wherever it is printed. `therock.rs` reads it through `pub(super)` items; it reads the provenance, host-note and `venv_python_path` helpers back from `therock.rs`. - [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. - [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK` and the background-helper fixtures `runtime_state` and `set_helper_facts`. - [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_serve_command_safety`, the four rules a model-proposed `serve` must satisfy (no public bind, no CPU device, a loopback host, `--managed` without `--foreground`), and `validate_chat_rocm_command_safety`, which applies them to a `serve` argv and also refuses a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_serve_command_safety` and `validate_chat_rocm_command_safety` here instead of through `chat`, and keeps no copy of the serve rules. `chat.rs`'s child modules call into it and `chat.rs` re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). From df5b61841bd2f6f18ff93ef9eab7b26146d7e4a7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:19:16 +0000 Subject: [PATCH 03/11] test(e2e): pin that a ROCm 10 preview still runs a uv that is already present A preview of a ROCm 10 or newer wheel has uv pip compile choose the torch and torchvision versions, and the plan's package specs, runtime key and command are made of them. Add runtime-27: with a uv present (the recording stand-in named by ROCM_CLI_UV_BINARY, answering the compile with the fixture's four pins) the preview runs it once, for the compile and nothing else, prints the resolved plan and reports no skipped step. It passes on the current code and is the half of the contract a later change must keep when a preview stops fetching uv on a host that has none. The stand-in gains an opt-in FAKE_UV_DRAIN_STDIN, because the CLI writes the requirements to uv's stdin before it reads the answer and a stand-in that exits without reading can fail that write. The two host setups that put an empty PATH and a cache folder of their own in the scenario now share one helper. Signed-off-by: Jussi Elo --- .../features/runtime_setup.feature | 23 ++++++ tests/e2e-cucumber/src/bin/fake-uv.rs | 12 +++ tests/e2e-cucumber/tests/e2e/therock_steps.rs | 74 ++++++++++++++++--- 3 files changed, 98 insertions(+), 11 deletions(-) diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index bc1b24ef42..a698912fd5 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -578,3 +578,26 @@ Feature: Runtime configuration Then the adoption is refused And the refusal asks for the environment folder with --root And no runtime is registered + + # A preview of a ROCm 10 or newer wheel has `uv pip compile` choose the torch + # and torchvision versions: nothing else does, and the plan's package specs, its + # runtime key and its command are made of them. When a `uv` is already there the + # preview must still run it, so the plan is the one a real install would print. + # This pins that half of the contract next to runtime-28, where no `uv` is there, + # so that skipping the step on a clean host cannot become skipping it everywhere. + # + # The recording `uv` is the one named by `ROCM_CLI_UV_BINARY`, so it counts as + # present, and it answers `uv pip compile` with the fixture's four pinned + # versions. Its log is the evidence the compile ran, once, and the plan + # carries the versions it printed. + # + # Linux only, for the same empty-`PATH` host as runtime-23 (the Python the + # preview would prepare is not what this scenario is about). + @id:runtime-install-preview-of-rocm-10-uses-the-uv-that-is-present @requires-os:linux + Scenario: runtime-27 - Previewing a ROCm 10 SDK install with a uv already present resolves the package versions with it + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a host with no usable Python and a recording uv + And the recording uv answers uv pip compile with the ROCm 10 fixture's versions + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 + Then the preview requests the gfx1200 device extras + And the preview resolved the versions with the uv that was present diff --git a/tests/e2e-cucumber/src/bin/fake-uv.rs b/tests/e2e-cucumber/src/bin/fake-uv.rs index 3227ae5abc..0bd0a4860e 100644 --- a/tests/e2e-cucumber/src/bin/fake-uv.rs +++ b/tests/e2e-cucumber/src/bin/fake-uv.rs @@ -14,10 +14,22 @@ //! arguments it was given (unset: nothing is written). A scenario that must //! show `uv` was never run asserts the file does not exist. //! +//! `FAKE_UV_DRAIN_STDIN` — when set, read standard input to its end before +//! answering (unset: stdin is left alone). The CLI writes the requirements to +//! `uv pip compile -` on stdin before it reads anything back, so a stand-in that +//! answers and exits without reading can make that write fail on a closed pipe, +//! depending on timing. Unset by default because a scenario that never pipes +//! anything in may leave stdin attached to something that never ends. +//! //! A Rust binary rather than a shell script so the scenarios run on Windows //! too. fn main() { + if std::env::var_os("FAKE_UV_DRAIN_STDIN").is_some() { + use std::io::Read as _; + let mut sink = Vec::new(); + let _ = std::io::stdin().read_to_end(&mut sink); + } if let Some(log) = std::env::var_os("FAKE_UV_LOG").map(std::path::PathBuf::from) { use std::io::Write as _; let args: Vec = std::env::args().skip(1).collect(); diff --git a/tests/e2e-cucumber/tests/e2e/therock_steps.rs b/tests/e2e-cucumber/tests/e2e/therock_steps.rs index d07a243d34..61c4999e9c 100644 --- a/tests/e2e-cucumber/tests/e2e/therock_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/therock_steps.rs @@ -269,6 +269,24 @@ const SCRATCH_CACHE_DIR: &str = "scratch-cache"; /// for a reason it does not name, so they are refused up front. #[given("a host with no usable Python and a recording uv")] async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { + let scenario_root = host_without_python(world); + let uv = scenario_root + .join("recording-uv-bin") + .join(e2e_cucumber::fixture_binary::fixture_target_name("uv")); + crate::install_fixture_binary("fake-uv", &uv); + world + .command_env + .push(("ROCM_CLI_UV_BINARY", uv.into_os_string())); + world.command_env.push(( + "FAKE_UV_LOG", + scenario_root.join(UV_INVOCATION_LOG).into_os_string(), + )); +} + +/// What the hosts these steps build share: `PATH` is one empty folder, so no Python is found +/// and none has been saved, and the CLI has a cache folder of its own. Returns +/// the scenario root. +fn host_without_python(world: &mut E2eWorld) -> std::path::PathBuf { for var in [ "ROCM_CLI_PYTHON", "ROCM_CLI_DISABLE_MANAGED_PYTHON_BOOTSTRAP", @@ -282,10 +300,6 @@ async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { let scenario_root = root(world).to_path_buf(); let empty_path = scenario_root.join("no-python-on-path"); std::fs::create_dir_all(&empty_path).expect("failed to create the empty PATH folder"); - let uv = scenario_root - .join("recording-uv-bin") - .join(e2e_cucumber::fixture_binary::fixture_target_name("uv")); - crate::install_fixture_binary("fake-uv", &uv); world .command_env .push(("PATH", empty_path.into_os_string())); @@ -298,13 +312,7 @@ async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { world .command_env .push(("ROCM_CLI_CACHE_DIR", scratch_cache.into_os_string())); - world - .command_env - .push(("ROCM_CLI_UV_BINARY", uv.into_os_string())); - world.command_env.push(( - "FAKE_UV_LOG", - scenario_root.join(UV_INVOCATION_LOG).into_os_string(), - )); + scenario_root } #[given("a canonical release pip index fixture and a ROCm 10 pip index fixture")] @@ -1300,6 +1308,50 @@ async fn preview_refused_for_python_version(world: &mut E2eWorld) { assert_no_python_prepared(world); } +/// The four pins `uv pip compile` would print for the ROCm 10 fixture, one +/// `name==version` per line, which the CLI reads back as the chosen stack. +#[given("the recording uv answers uv pip compile with the ROCm 10 fixture's versions")] +async fn recording_uv_answers_compile(world: &mut E2eWorld) { + let pins = [ + ("rocm", NEXT_ROCM_VERSION), + ("torch", NEXT_TORCH_VERSION), + ("torchvision", NEXT_TORCHVISION_VERSION), + ("torchaudio", NEXT_TORCHAUDIO_VERSION), + ] + .map(|(name, version)| format!("{name}=={version}")) + .join("\n"); + world.command_env.push(("FAKE_UV_STDOUT", pins.into())); + // The CLI pipes the requirements into `uv pip compile -`; see the stand-in. + world.command_env.push(("FAKE_UV_DRAIN_STDIN", "1".into())); +} + +/// A `uv` that is present is still used: it ran `uv pip compile`, once and for +/// nothing else, and the preview did not report the step as skipped. The log is +/// the evidence, since the plan's versions alone could not tell a compile from a +/// guess. +#[then("the preview resolved the versions with the uv that was present")] +async fn preview_resolved_with_present_uv(world: &mut E2eWorld) { + let log_path = root(world).join(UV_INVOCATION_LOG); + let log = std::fs::read_to_string(&log_path).unwrap_or_else(|error| { + panic!( + "the recording uv has no log at {}, so the preview never ran it: {error}", + log_path.display() + ) + }); + let runs: Vec<&str> = log.lines().collect(); + assert!( + matches!(runs.as_slice(), [run] if run.starts_with("pip compile ")), + "the preview should run uv once, for `uv pip compile`, and for nothing else:\n{log}" + ); + let out = stdout(world); + for unresolved in ["resolution_note:", "not resolved by this preview"] { + assert!( + !out.contains(unresolved), + "the preview ran uv but reported the step as skipped ({unresolved:?}):\n{out}" + ); + } +} + #[then("the preview resolves the canonical release pip index")] async fn preview_resolves_canonical_pip_index(world: &mut E2eWorld) { let base = current_pip_base(world); From 30620a5b1199b1b73ac67c6080df7db18821ecbe Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:48:36 +0000 Subject: [PATCH 04/11] fix(install): preview a ROCm 10 SDK install without fetching uv On the ROCm 10 and newer wheel layout, rocm install sdk --dry-run (and the install_sdk_dry_run assistant tool) resolved the torch and torchvision versions with uv pip compile through the same resolver a real install uses, which acquires the managed uv: on a clean host the preview downloaded uv and created the data folder's tools folder although it promises to change nothing. The canonical layout needs no uv and was not affected. The resolver now takes a UvAccess. Acquire is what a real install and rocm update pass, and they decide and call exactly what they did before. PresentOnly is the preview's: it runs a uv that is already there (the binary ROCM_CLI_UV_BINARY names, or the managed copy a previous install left) and, with none, stops before the compile and returns an UnresolvedPipRuntime holding what needs no uv (the ROCm version, the torchaudio pin, the requirements uv would be given) and no package versions. The preview prints that plan through the same writer as every other plan, with target, runtime_key, package_specs, command and manifest marked not resolved by this preview, and says why in a resolution_note line and a progress line, in the voice of python_note. It cannot say whether the published stack installs together either, and the note says that. The decision is one pure function, tested over both layouts, both accesses and a uv present or not; a second test runs the real resolution against a served index and a stand-in uv and counts what each row looked for, acquired and ran. Scenario runtime-28 asserts the note together with the state on a host with no uv: no tools folder, no tool archive, no saved config and no torch or torchvision version in the output. The previews in therock-next-02, -10, -11 and -13 that print resolved specs start from an installed uv. Signed-off-by: Jussi Elo --- apps/rocm/src/therock.rs | 95 +- apps/rocm/src/therock/resolve.rs | 6 +- apps/rocm/src/therock/resolve/wheel.rs | 1123 +++++++++++++++-- apps/rocm/src/therock/wheel_plan.rs | 246 +++- docs/architecture.md | 4 +- .../features/runtime_setup.feature | 28 + .../features/therock_next_generation.feature | 11 + tests/e2e-cucumber/tests/e2e/therock_steps.rs | 99 +- 8 files changed, 1435 insertions(+), 177 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index c05b12b1d2..3b119d5de9 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -45,8 +45,9 @@ use self::python::{ ensure_uv_venv, managed_python_version, python_for_install, python_requirement, }; use self::resolve::{ - AggregateDeviceTarget, canonical_auto_select_needs_next_python, resolve_pip_runtime, - resolve_tarball_artifact, unverified_torchaudio_note, wheel_runtime_composition, + AggregateDeviceTarget, PipRuntimePlan, canonical_auto_select_needs_next_python, + preview_pip_runtime, resolve_pip_runtime, resolve_tarball_artifact, unverified_torchaudio_note, + wheel_runtime_composition, }; pub(crate) use self::source::RuntimeVersionSelector; #[cfg(test)] @@ -63,7 +64,10 @@ pub(crate) use self::version::runtime_version_display; use self::version::{ parse_host_version, parse_version_for_ordering, runtime_version_build_date, version_relation, }; -use self::wheel_plan::{WheelPlan, write_dry_run_tail, write_wheel_plan}; +use self::wheel_plan::{ + PlanStack, ResolvedStack, UNRESOLVED_PROGRESS, UnresolvedPreview, WheelPlan, + unresolved_wheel_preview, write_dry_run_tail, write_wheel_plan, +}; #[cfg(test)] pub(crate) use crate::runtime_registry::RocmSdkLibraryProbe; #[cfg(test)] @@ -388,15 +392,50 @@ fn install_wheel_runtime( "Checking TheRock {} packages for this AMD GPU...", channel.as_str() )); - let resolution = resolve_pip_runtime( - paths, - channel, - recovered_family_override.as_deref(), - &wheel_compatibility, - version_selector, - layout_override, - include_devel, - )?; + // A preview must not fetch `uv` either: on the ROCm 10 layout resolving the + // package versions runs `uv pip compile`, and a real install gets the managed + // `uv` for that by downloading it. A preview uses a `uv` that is already there + // and otherwise stops with what it can know without one, which says so. A real + // install and an update resolve, and so fetch, exactly as they always did. + let resolution = if dry_run { + match preview_pip_runtime( + paths, + channel, + recovered_family_override.as_deref(), + &wheel_compatibility, + version_selector, + layout_override, + include_devel, + )? { + PipRuntimePlan::Resolved(resolution) => resolution, + PipRuntimePlan::NeedsUv(unresolved) => { + progress_line(UNRESOLVED_PROGRESS); + let host_version_newer = host_rocm_version_newer_than(&unresolved.rocm_version); + return Ok(SdkInstallResult::plan(unresolved_wheel_preview( + &UnresolvedPreview { + channel, + version_selector, + unresolved: &unresolved, + device_target_override, + prefix: prefix.as_deref(), + host_version_newer: host_version_newer.as_deref(), + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ))); + } + } + } else { + resolve_pip_runtime( + paths, + channel, + recovered_family_override.as_deref(), + &wheel_compatibility, + version_selector, + layout_override, + include_devel, + )? + }; let device_target = device_target_override.map_or_else( || resolution.device_target.clone(), |target| { @@ -425,16 +464,10 @@ fn install_wheel_runtime( // filesystem scan for a legacy ROCm, and the summary block and the visible // note report the same answer about the same resolved version. let host_version_newer = host_rocm_version_newer_than(&resolution.latest_version); - let no_wheel_warning = repo_version_without_wheels( + let no_wheel_warning = no_wheel_warning_for( resolution.newest_repo_version.as_deref(), &resolution.latest_version, - ) - .map(|newest| { - no_wheel_warning_message( - &newest, - &runtime_version_display(&resolution.latest_version), - ) - }); + ); let plan = WheelPlan { channel, version_selector, @@ -445,20 +478,23 @@ fn install_wheel_runtime( family_source: &resolution.family_source, device_target: &device_target, host_version_newer: host_version_newer.as_deref(), - compatibility_key: &resolution.package_versions.compatibility_key, - install_root: &install_root, - runtime_key: &runtime_key, python: &python, wheel_compatibility: &wheel_compatibility, version_resolution_specs: resolution.version_resolution_specs.as_deref(), torchaudio_pin: resolution.torchaudio_pin.as_ref(), - package_specs: &wheel_composition.package_specs, no_wheel_warning: no_wheel_warning.as_deref(), + stack: PlanStack::Resolved(ResolvedStack { + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: &runtime_key, + package_specs: &wheel_composition.package_specs, + manifest_path: &manifest_path, + }), }; let mut output = String::new(); write_wheel_plan(&mut output, &plan); if dry_run { - write_dry_run_tail(&mut output, &plan, &manifest_path); + write_dry_run_tail(&mut output, &plan); return Ok(SdkInstallResult::plan(output)); } @@ -719,6 +755,15 @@ fn no_wheel_warning_message(newest: &str, resolved_display: &str) -> String { ) } +/// The `warning` surfaced when the repository's newest version has no installable +/// PyTorch wheels, or `None` when there is no such gap: [`repo_version_without_wheels`] +/// decides whether there is one and [`no_wheel_warning_message`] words it. One call +/// for a plan that is resolved and one that is not, so they cannot disagree. +fn no_wheel_warning_for(newest_repo: Option<&str>, resolved_version: &str) -> Option { + repo_version_without_wheels(newest_repo, resolved_version) + .map(|newest| no_wheel_warning_message(&newest, &runtime_version_display(resolved_version))) +} + /// The repo's newest version when it is strictly newer than the version we are /// about to install — i.e. the newest release exists in the index but has no /// installable PyTorch wheels for this Python/platform, so an older one is used. diff --git a/apps/rocm/src/therock/resolve.rs b/apps/rocm/src/therock/resolve.rs index 0dddb3ef51..05e7f2a7d5 100644 --- a/apps/rocm/src/therock/resolve.rs +++ b/apps/rocm/src/therock/resolve.rs @@ -14,7 +14,7 @@ pub(super) use self::tarball::{resolve_tarball_artifact, resolve_tarball_artifac #[cfg(test)] pub(super) use self::wheel::select_rocm_version; pub(super) use self::wheel::{ - AggregateDeviceTarget, TorchaudioPin, canonical_auto_select_needs_next_python, - resolve_pip_runtime, resolve_pip_runtime_with_timeout, unverified_torchaudio_note, - wheel_runtime_composition, + AggregateDeviceTarget, PipRuntimePlan, TorchaudioPin, UnresolvedPipRuntime, + canonical_auto_select_needs_next_python, preview_pip_runtime, resolve_pip_runtime, + resolve_pip_runtime_with_timeout, unverified_torchaudio_note, wheel_runtime_composition, }; diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index cf5c1f1a92..03b2ec15f4 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -15,7 +15,11 @@ //! `torch`, `torchvision` and `torchaudio` listings agree; on the next layout //! `torchaudio_pin` pins a torchaudio first (the hardware-verified release when //! it is published) and `resolve_published_pip_package_versions` has -//! `uv pip compile` choose the rest. `wheel_runtime_composition` turns a +//! `uv pip compile` choose the rest. That compile is the one step that may fetch +//! `uv`: a real install and an update (`resolve_pip_runtime`) acquire it, and a +//! preview (`preview_pip_runtime`) runs one that is already present and +//! otherwise stops with an `UnresolvedPipRuntime`, which is decided in +//! `plan_version_resolution`. `wheel_runtime_composition` turns a //! resolution into the exact package specs (`therock_pip_package_specs`) an //! install gives `uv` and records in the manifest, and //! `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` @@ -48,10 +52,11 @@ use super::super::version::{ use crate::runtime_registry::WheelRuntimeComposition; use anyhow::{Context, Result, bail}; use rocm_core::{ - AppPaths, detect_host_gfx_target, ensure_uv_binary, extract_first_gfx_token, + AppPaths, detect_host_gfx_target, ensure_uv_binary, extract_first_gfx_token, find_uv_binary, normalize_therock_family, runtime_is_windows, uv_command_env, }; use std::io::Write; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::Duration; @@ -480,6 +485,113 @@ pub(in crate::therock) struct PipRuntimeResolution { pub(in crate::therock) published_device_targets: Vec, } +/// Whether a resolution may fetch `uv` to choose the ROCm 10 package versions. +/// +/// Only the ROCm 10 (`next`) layout needs `uv`: it has `uv pip compile` pick the +/// `torch` and `torchvision` builds. That is the one step of resolution that can +/// download the managed `uv` and create the data folder's `tools` folder. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::therock) enum UvAccess { + /// A real install or an update: take the managed `uv`, downloading it if no + /// copy is installed yet. This is what resolution has always done. + Acquire, + /// A preview: use a `uv` that is already there and never fetch one. + PresentOnly, +} + +/// What a resolution may do about `uv`: how far it may go ([`UvAccess`]) and the two +/// things it can ask for. The real ones, `find_uv_binary` and `ensure_uv_binary`, +/// are built where a resolution starts; they are fields so that a test can count +/// the calls instead of reaching the network. +struct UvPolicy<'a> { + access: UvAccess, + /// A `uv` that is already there, without acquiring one. + find_present: &'a dyn Fn() -> Option, + /// The managed `uv`, downloading it when none is installed. + acquire: &'a dyn Fn() -> Result, +} + +/// How a resolution chooses the package versions, given [`UvAccess`] and whether a +/// `uv` is present. A pure function of those inputs: see [`plan_version_resolution`]. +#[derive(Debug, Clone, PartialEq, Eq)] +enum VersionResolution { + /// The canonical layout: read each package's simple index. No `uv`. + ScrapeIndexes, + /// The ROCm 10 layout: run `uv pip compile` with this `uv`. + Compile(UvChoice), + /// The ROCm 10 layout with no `uv` to run and none to fetch: stop with what is + /// known without one, and leave the versions `uv` would choose unresolved. + Skip, +} + +/// Which `uv` the compile step runs. +#[derive(Debug, Clone, PartialEq, Eq)] +enum UvChoice { + /// Ask `ensure_uv_binary`, which downloads the managed `uv` when none is installed. + Acquire, + /// A `uv` that was already there. + Existing(PathBuf), +} + +/// Decide how the versions are chosen, without doing it. +/// +/// `find_present` looks for a `uv` that is already there. It runs only when the +/// answer depends on it: never for the canonical layout, which needs no `uv`, and +/// never for [`UvAccess::Acquire`], whose `ensure_uv_binary` does its own looking. +/// So a real install and an update decide, and call, exactly what they did before a +/// preview could skip the step, and no answer for them is [`VersionResolution::Skip`]. +fn plan_version_resolution( + layout: SourceLayout, + access: UvAccess, + find_present: impl FnOnce() -> Option, +) -> VersionResolution { + match (layout, access) { + (SourceLayout::Canonical, _) => VersionResolution::ScrapeIndexes, + (SourceLayout::Next, UvAccess::Acquire) => VersionResolution::Compile(UvChoice::Acquire), + (SourceLayout::Next, UvAccess::PresentOnly) => find_present() + .map_or(VersionResolution::Skip, |uv| { + VersionResolution::Compile(UvChoice::Existing(uv)) + }), + } +} + +/// What a resolution knows once it has been asked for with a given [`UvAccess`]. +#[derive(Debug, Clone)] +pub(in crate::therock) enum PipRuntimePlan { + Resolved(PipRuntimeResolution), + /// A preview of the ROCm 10 layout on a host with no `uv`. Only + /// [`UvAccess::PresentOnly`] yields it. + NeedsUv(UnresolvedPipRuntime), +} + +/// A ROCm 10 resolution stopped before `uv pip compile`: everything chosen without +/// `uv`, and none of what depends on it. +/// +/// Known: the family, the device target, the index, the `rocm` version and the +/// torchaudio pin, and the requirement lines `uv` would be asked to resolve. Not +/// known: the `torch` and `torchvision` versions, which only `uv pip compile` +/// produces, nor whether the published stack is jointly installable. Everything +/// built from them (the package specs, the runtime key, the install folder and +/// manifest the key names) is therefore unknown too, which is why this carries no +/// [`TheRockPipPackageVersions`]: a value that cannot be made cannot be mistaken +/// for one. +#[derive(Debug, Clone)] +pub(in crate::therock) struct UnresolvedPipRuntime { + pub(in crate::therock) family: String, + pub(in crate::therock) family_source: String, + pub(in crate::therock) index_url: String, + pub(in crate::therock) layout: SourceLayout, + /// The `rocm` version the install would be for. + pub(in crate::therock) rocm_version: String, + pub(in crate::therock) newest_repo_version: Option, + /// The requirement lines `uv pip compile` would be given, as + /// [`PipRuntimeResolution::version_resolution_specs`] holds them once it ran. + pub(in crate::therock) version_resolution_specs: Vec, + pub(in crate::therock) torchaudio_pin: TorchaudioPin, + pub(in crate::therock) device_target: AggregateDeviceTarget, + pub(in crate::therock) published_device_targets: Vec, +} + #[derive(Debug, Clone, Eq, PartialEq)] pub(in crate::therock) struct TheRockPipPackageVersions { rocm: String, @@ -618,6 +730,10 @@ pub(in crate::therock) fn resolve_pip_runtime( /// `layout_override` is how an update keeps a runtime on the stream it was /// installed from; a fresh install passes `None` and lets the request decide. +/// +/// Always [`UvAccess::Acquire`]: a real install and an update take the managed `uv` +/// when the ROCm 10 layout needs one, as they always have. A preview asks +/// [`preview_pip_runtime`] instead. #[allow(clippy::too_many_arguments)] pub(in crate::therock) fn resolve_pip_runtime_with_timeout( paths: &AppPaths, @@ -629,6 +745,69 @@ pub(in crate::therock) fn resolve_pip_runtime_with_timeout( include_devel: bool, download_timeout_secs: Option, ) -> Result { + match resolve_pip_runtime_plan( + paths, + channel, + family_override, + wheel_compatibility, + version_selector, + layout_override, + include_devel, + download_timeout_secs, + UvAccess::Acquire, + )? { + PipRuntimePlan::Resolved(resolution) => Ok(resolution), + // `plan_version_resolution` answers `Skip` only to `UvAccess::PresentOnly`, + // and a test holds it to that, so this is a broken invariant, not a state. + PipRuntimePlan::NeedsUv(_) => { + bail!("internal error: a resolution that may acquire uv stopped for want of one") + } + } +} + +/// [`resolve_pip_runtime`] for a preview: it never downloads `uv` or creates the +/// folders that hold it. A `uv` that is already there is used, so the answer is the +/// resolution a real install would reach; with none, a ROCm 10 request comes back as +/// [`PipRuntimePlan::NeedsUv`], holding what `uv` is not needed for. +pub(in crate::therock) fn preview_pip_runtime( + paths: &AppPaths, + channel: TheRockChannel, + family_override: Option<&str>, + wheel_compatibility: &WheelCompatibility, + version_selector: Option<&RuntimeVersionSelector>, + layout_override: Option, + include_devel: bool, +) -> Result { + resolve_pip_runtime_plan( + paths, + channel, + family_override, + wheel_compatibility, + version_selector, + layout_override, + include_devel, + None, + UvAccess::PresentOnly, + ) +} + +#[allow(clippy::too_many_arguments)] +fn resolve_pip_runtime_plan( + paths: &AppPaths, + channel: TheRockChannel, + family_override: Option<&str>, + wheel_compatibility: &WheelCompatibility, + version_selector: Option<&RuntimeVersionSelector>, + layout_override: Option, + include_devel: bool, + download_timeout_secs: Option, + uv_access: UvAccess, +) -> Result { + let uv = UvPolicy { + access: uv_access, + find_present: &|| find_uv_binary(paths), + acquire: &|| ensure_uv_binary(paths), + }; let family_resolution = resolve_family(paths, family_override)?; let layout = match layout_override { Some(layout) => layout, @@ -680,6 +859,7 @@ pub(in crate::therock) fn resolve_pip_runtime_with_timeout( version_selector, include_devel, download_timeout_secs, + &uv, ) .with_context(|| { format!( @@ -704,7 +884,8 @@ fn resolve_pip_runtime_from_index( version_selector: Option<&RuntimeVersionSelector>, include_devel: bool, download_timeout_secs: Option, -) -> Result { + uv: &UvPolicy<'_>, +) -> Result { let index_url = source.index_url.as_str(); let mut version_resolution_specs = None; let mut torchaudio_selection = None; @@ -720,99 +901,6 @@ fn resolve_pip_runtime_from_index( "release channel only installs stable TheRock wheel versions, but no stable `rocm` package versions were found in {index_url}; try `rocm install sdk --channel release --format tarball` for stable release artifacts, or use `--channel nightly --format wheel` for preview builds" ); } - let package_versions = if matches!(source.layout, SourceLayout::Next) { - let device_target = match &source.device_target { - AggregateDeviceTarget::Exact(target) => target.as_str(), - AggregateDeviceTarget::Undetermined(reason) => { - bail!( - "cannot resolve ROCm X package metadata without an exact device target: {reason}" - ) - } - }; - let rocm_version = select_rocm_version(channel, &rocm_versions, version_selector) - .with_context(|| { - let requested = version_selector.map_or_else( - || "latest compatible version".to_owned(), - RuntimeVersionSelector::describe, - ); - format!("no TheRock rocm package was found for {requested} in {index_url}") - })?; - // Chosen here, not by `uv`: see `published_pip_requirements`. - let torchaudio_versions = load_simple_index_versions( - paths, - index_url, - "torchaudio", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let pin = torchaudio_pin( - &torchaudio_versions, - &rocm_version, - rocm_engine_vllm::verified_torchaudio_release(&rocm_version), - index_url, - )?; - // The canonical branch below budgets one `download_timeout_secs` per - // package (rocm, then torch, torchvision, torchaudio: 4 sequential HTTP - // fetches). `uv pip compile` still fetches metadata for all four - // packages in one subprocess call — the torchaudio listing above only - // chose its version — so it needs a comparable aggregate budget, not - // the single-fetch one: reusing the bare per-fetch value here would make - // a startup check that budgets 2s per fetch reliably time out a call - // doing 4 fetches' worth of work. That listing is one more fetch on top, - // with its own single-fetch budget. - let requirements = - published_pip_requirements(&rocm_version, &pin.version, device_target, include_devel); - let versions = resolve_published_pip_package_versions( - paths, - index_url, - &requirements, - wheel_compatibility, - download_timeout_secs.map(|secs| secs.saturating_mul(4)), - )?; - version_resolution_specs = Some(requirements); - torchaudio_selection = Some(pin); - versions - } else { - let torch_versions = load_simple_index_versions( - paths, - index_url, - "torch", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let torchvision_versions = load_simple_index_versions( - paths, - index_url, - "torchvision", - Some(wheel_compatibility), - download_timeout_secs, - )?; - let torchaudio_versions = load_simple_index_versions( - paths, - index_url, - "torchaudio", - Some(wheel_compatibility), - download_timeout_secs, - )?; - select_matching_pip_package_versions( - channel, - &rocm_versions, - &torch_versions, - &torchvision_versions, - &torchaudio_versions, - version_selector, - ) - .with_context(|| { - let requested = version_selector.map_or_else( - || "latest compatible version".to_owned(), - RuntimeVersionSelector::describe, - ); - format!( - "no mutually compatible TheRock rocm, torch, torchvision, and torchaudio versions were found for {requested} in {index_url}" - ) - })? - }; - let latest_version = package_versions.rocm.clone(); // The repo's newest version for this channel, ignoring wheel availability. // Only meaningful when we auto-selected "latest" (no explicit request), so a // caller can warn when that newest version has no installable wheels. @@ -823,7 +911,126 @@ fn resolve_pip_runtime_from_index( } else { None }; - Ok(PipRuntimeResolution { + let package_versions = match plan_version_resolution(source.layout, uv.access, uv.find_present) + { + VersionResolution::ScrapeIndexes => { + let torch_versions = load_simple_index_versions( + paths, + index_url, + "torch", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let torchvision_versions = load_simple_index_versions( + paths, + index_url, + "torchvision", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let torchaudio_versions = load_simple_index_versions( + paths, + index_url, + "torchaudio", + Some(wheel_compatibility), + download_timeout_secs, + )?; + select_matching_pip_package_versions( + channel, + &rocm_versions, + &torch_versions, + &torchvision_versions, + &torchaudio_versions, + version_selector, + ) + .with_context(|| { + let requested = version_selector.map_or_else( + || "latest compatible version".to_owned(), + RuntimeVersionSelector::describe, + ); + format!( + "no mutually compatible TheRock rocm, torch, torchvision, and torchaudio versions were found for {requested} in {index_url}" + ) + })? + } + resolution @ (VersionResolution::Compile(_) | VersionResolution::Skip) => { + let device_target = match &source.device_target { + AggregateDeviceTarget::Exact(target) => target.as_str(), + AggregateDeviceTarget::Undetermined(reason) => { + bail!( + "cannot resolve ROCm X package metadata without an exact device target: {reason}" + ) + } + }; + let rocm_version = select_rocm_version(channel, &rocm_versions, version_selector) + .with_context(|| { + let requested = version_selector.map_or_else( + || "latest compatible version".to_owned(), + RuntimeVersionSelector::describe, + ); + format!("no TheRock rocm package was found for {requested} in {index_url}") + })?; + // Chosen here, not by `uv`: see `published_pip_requirements`. + let torchaudio_versions = load_simple_index_versions( + paths, + index_url, + "torchaudio", + Some(wheel_compatibility), + download_timeout_secs, + )?; + let pin = torchaudio_pin( + &torchaudio_versions, + &rocm_version, + rocm_engine_vllm::verified_torchaudio_release(&rocm_version), + index_url, + )?; + let requirements = published_pip_requirements( + &rocm_version, + &pin.version, + device_target, + include_devel, + ); + let VersionResolution::Compile(choice) = resolution else { + // No `uv` to run and none to fetch: report what is known, and + // leave the versions `uv` would choose to the install. + return Ok(PipRuntimePlan::NeedsUv(UnresolvedPipRuntime { + family: family_resolution.family.clone(), + family_source: family_resolution.source.clone(), + index_url: index_url.to_owned(), + layout: source.layout, + rocm_version, + newest_repo_version, + version_resolution_specs: requirements, + torchaudio_pin: pin, + device_target: source.device_target.clone(), + published_device_targets: source.published_device_targets.clone(), + })); + }; + // The canonical branch above budgets one `download_timeout_secs` per + // package (rocm, then torch, torchvision, torchaudio: 4 sequential HTTP + // fetches). `uv pip compile` still fetches metadata for all four + // packages in one subprocess call — the torchaudio listing above only + // chose its version — so it needs a comparable aggregate budget, not + // the single-fetch one: reusing the bare per-fetch value here would make + // a startup check that budgets 2s per fetch reliably time out a call + // doing 4 fetches' worth of work. That listing is one more fetch on top, + // with its own single-fetch budget. + let compile_uv = uv_for_compile(choice, uv.acquire)?; + let versions = resolve_published_pip_package_versions( + paths, + &compile_uv, + index_url, + &requirements, + wheel_compatibility, + download_timeout_secs.map(|secs| secs.saturating_mul(4)), + )?; + version_resolution_specs = Some(requirements); + torchaudio_selection = Some(pin); + versions + } + }; + let latest_version = package_versions.rocm.clone(); + Ok(PipRuntimePlan::Resolved(PipRuntimeResolution { family: family_resolution.family.clone(), family_source: family_resolution.source.clone(), index_url: index_url.to_owned(), @@ -835,7 +1042,18 @@ fn resolve_pip_runtime_from_index( package_versions, device_target: source.device_target.clone(), published_device_targets: source.published_device_targets.clone(), - }) + })) +} + +/// The `uv` the compile step runs: the one already there, or the managed `uv` that +/// `acquire` (`ensure_uv_binary`) fetches when none is installed. +fn uv_for_compile(choice: UvChoice, acquire: impl FnOnce() -> Result) -> Result { + match choice { + UvChoice::Existing(uv) => Ok(uv), + UvChoice::Acquire => { + acquire().context("failed to acquire uv for ROCm X metadata resolution") + } + } } pub(in crate::therock) fn select_rocm_version( @@ -937,19 +1155,22 @@ fn parse_uv_compiled_package_versions(output: &str) -> Result, ) -> Result { - let uv = - ensure_uv_binary(paths).context("failed to acquire uv for ROCm X metadata resolution")?; let python_version = uv_python_version(compatibility)?; let python_platform = uv_python_platform(compatibility)?; let requirements = format!("{}\n", requirements.join("\n")); - let mut child = Command::new(&uv) + let mut child = Command::new(uv) .args([ "pip", "compile", @@ -2322,4 +2543,704 @@ mod tests { assert!(hint.contains("--format tarball")); } } + + // ---- who may fetch `uv` ------------------------------------------------- + + /// Asks [`plan_version_resolution`] for one cell of the matrix and reports + /// the answer together with how many times a present `uv` was looked for. + fn decide( + layout: SourceLayout, + access: UvAccess, + uv_present: bool, + ) -> (VersionResolution, usize) { + let looked = std::cell::Cell::new(0); + let decision = plan_version_resolution(layout, access, || { + looked.set(looked.get() + 1); + uv_present.then(|| PathBuf::from("/opt/uv")) + }); + (decision, looked.get()) + } + + /// The state matrix of the decision: both layouts, both accesses, a `uv` + /// present or not. Only the ROCm 10 layout needs `uv`; a real install and an + /// update (`Acquire`) always go on to acquire it and never ask whether one is + /// present, because `ensure_uv_binary` does its own looking; a preview + /// (`PresentOnly`) asks, and with none present stops. `Skip` therefore exists + /// for exactly one row. + #[test] + fn the_version_step_is_decided_by_layout_access_and_a_present_uv_alone() { + use SourceLayout::{Canonical, Next}; + use UvAccess::{Acquire, PresentOnly}; + let existing = || VersionResolution::Compile(UvChoice::Existing(PathBuf::from("/opt/uv"))); + let rows = [ + // (layout, access, uv present, decision, times a present uv was looked for) + ( + Canonical, + Acquire, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + Acquire, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Next, + Acquire, + false, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + ( + Next, + Acquire, + true, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + (Next, PresentOnly, false, VersionResolution::Skip, 1), + (Next, PresentOnly, true, existing(), 1), + ]; + for (layout, access, uv_present, expected, looked) in rows { + let (decision, calls) = decide(layout, access, uv_present); + let row = format!("{layout:?}, {access:?}, uv present: {uv_present}"); + assert_eq!(decision, expected, "decision for {row}"); + assert_eq!(calls, looked, "times a present uv was looked for in {row}"); + } + } + + /// The compile step runs the `uv` it was given, and asks for one only on + /// `Acquire`, with the context a failed fetch has always carried. + #[test] + fn the_compile_step_acquires_uv_only_when_told_to() { + let acquired = std::cell::Cell::new(0); + let acquire = || { + acquired.set(acquired.get() + 1); + Ok(PathBuf::from("/managed/uv")) + }; + assert_eq!( + uv_for_compile(UvChoice::Acquire, acquire).unwrap(), + PathBuf::from("/managed/uv") + ); + assert_eq!(acquired.get(), 1); + assert_eq!( + uv_for_compile(UvChoice::Existing(PathBuf::from("/opt/uv")), acquire).unwrap(), + PathBuf::from("/opt/uv") + ); + assert_eq!(acquired.get(), 1, "a present uv is not acquired again"); + + let error = uv_for_compile(UvChoice::Acquire, || bail!("no network")).unwrap_err(); + assert_eq!( + format!("{error:#}"), + "failed to acquire uv for ROCm X metadata resolution: no network" + ); + } + + /// Serves fixed pages over HTTP until dropped. A request for a path it does + /// not hold is a 404. + #[cfg(unix)] + struct IndexServer { + base: String, + stop: std::sync::Arc, + thread: Option>, + } + + #[cfg(unix)] + impl IndexServer { + fn start(pages: Vec<(String, String)>) -> Self { + use std::io::{Read, Write}; + use std::sync::atomic::Ordering; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + listener.set_nonblocking(true).expect("nonblocking"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stopped = std::sync::Arc::clone(&stop); + let thread = std::thread::spawn(move || { + while !stopped.load(Ordering::SeqCst) { + let Ok((mut stream, _)) = listener.accept() else { + std::thread::sleep(Duration::from_millis(5)); + continue; + }; + let _ = stream.set_nonblocking(false); + let _ = stream.set_read_timeout(Some(Duration::from_secs(5))); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + match stream.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(read) => request.extend_from_slice(&buffer[..read]), + } + } + let request = String::from_utf8_lossy(&request); + let path = request.split_whitespace().nth(1).unwrap_or_default(); + let page = pages.iter().find(|(served, _)| served == path); + let (status, body) = match page { + Some((_, body)) => ("200 OK", body.as_str()), + None => ("404 Not Found", ""), + }; + let _ = write!( + stream, + "HTTP/1.1 {status}\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.flush(); + } + }); + Self { + base: format!("http://127.0.0.1:{port}"), + stop, + thread: Some(thread), + } + } + } + + #[cfg(unix)] + impl Drop for IndexServer { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + #[cfg(unix)] + fn wheel_links(package: &str, versions: &[&str]) -> String { + versions.iter().fold(String::new(), |mut html, version| { + let file = format!("{package}-{version}-py3-none-any.whl"); + html.push_str(&format!("{file}\n")); + html + }) + } + + /// One canonical and one ROCm 10 index, as the preview's resolution reads them. + #[cfg(unix)] + fn two_layout_index() -> IndexServer { + let mut pages = Vec::new(); + for (layout, stack) in [ + ( + "current", + [ + ("rocm", "7.10.0"), + ("torch", "2.9.0+rocm7.10.0"), + ("torchvision", "0.24.0+rocm7.10.0"), + ("torchaudio", "2.9.0+rocm7.10.0"), + ], + ), + ( + "next", + [ + ("rocm", "10.0.0"), + ("torch", "2.10.0+rocm10.0.0"), + ("torchvision", "0.25.0+rocm10.0.0"), + ("torchaudio", "2.9.1+rocm10.0.0"), + ], + ), + ] { + for (package, version) in stack { + pages.push(( + format!("/{layout}/{package}/"), + wheel_links(package, &[version]), + )); + } + } + IndexServer::start(pages) + } + + /// A `uv` that logs its arguments, reads the requirements it is piped, and + /// answers `uv pip compile` with the ROCm 10 stack above. + #[cfg(unix)] + fn write_logging_uv(dir: &Path) -> (PathBuf, PathBuf) { + let log = dir.join("uv-runs.log"); + let uv = dir.join("uv"); + std::fs::create_dir_all(dir).expect("create the uv folder"); + let script = format!( + "#!/bin/sh\necho \"$@\" >> '{}'\ncat > /dev/null\nprintf '%s\\n' 'rocm==10.0.0' 'torch==2.10.0+rocm10.0.0' 'torchvision==0.25.0+rocm10.0.0' 'torchaudio==2.9.1+rocm10.0.0'\n", + log.display() + ); + rocm_core::fake_executable::write_fake_executable(&uv, script) + .expect("plant the stand-in uv"); + (uv, log) + } + + /// What one cell of the matrix did: the answer, and what it touched. + #[cfg(unix)] + struct UvAccessRun { + plan: Result, + /// Times a present `uv` was looked for. + looked: usize, + /// Times the managed `uv` was acquired (downloaded, for a real one). + acquired: usize, + /// The arguments `uv` was run with, one run per line. + uv_runs: String, + /// Whether the managed tools folder, where a download goes, exists. + tools_folder: bool, + } + + /// Runs the real resolution against the served index for `layout`, with `uv` + /// "present" at the logging stand-in or not, and `acquire` answering for the + /// managed `uv` with that stand-in or with a failure. + #[cfg(unix)] + fn resolve_through_policy( + layout: SourceLayout, + access: UvAccess, + uv_present: bool, + acquire_succeeds: bool, + ) -> UvAccessRun { + let (root, paths) = super::super::super::test_support::test_paths("uv-access"); + let (uv, log) = write_logging_uv(&root.join("bin")); + let server = two_layout_index(); + let (index, python_tag) = match layout { + SourceLayout::Canonical => (format!("{}/current", server.base), "cp312"), + SourceLayout::Next => (format!("{}/next", server.base), "cp314"), + }; + let source = ResolvedAggregateWheelSource { + index_url: index, + layout, + device_target: AggregateDeviceTarget::Exact("gfx1200".to_owned()), + published_device_targets: vec!["gfx1200".to_owned()], + }; + let compatibility = WheelCompatibility { + python_tag: python_tag.to_owned(), + platform_tags: vec!["linux_x86_64".to_owned(), "any".to_owned()], + }; + let looked = std::cell::Cell::new(0); + let acquired = std::cell::Cell::new(0); + let find_present = || { + looked.set(looked.get() + 1); + uv_present.then(|| uv.clone()) + }; + let acquire = || { + acquired.set(acquired.get() + 1); + if acquire_succeeds { + Ok(uv.clone()) + } else { + bail!("the managed uv could not be downloaded") + } + }; + let policy = UvPolicy { + access, + find_present: &find_present, + acquire: &acquire, + }; + let plan = resolve_pip_runtime_from_index( + &paths, + TheRockChannel::Release, + &super::super::super::test_support::family_for_test("gfx1200", Some("gfx1200")), + &source, + &compatibility, + None, + false, + None, + &policy, + ); + let run = UvAccessRun { + plan, + looked: looked.get(), + acquired: acquired.get(), + uv_runs: std::fs::read_to_string(&log).unwrap_or_default(), + tools_folder: rocm_core::managed_tools_dir(&paths.data_dir).exists(), + }; + let _ = std::fs::remove_dir_all(&root); + run + } + + #[cfg(unix)] + fn stack_of(plan: &PipRuntimePlan) -> &TheRockPipPackageVersions { + match plan { + PipRuntimePlan::Resolved(resolution) => &resolution.package_versions, + PipRuntimePlan::NeedsUv(_) => { + panic!("the plan stopped before the versions were chosen") + } + } + } + + /// The resolution itself, every layout and access with a `uv` present or not, + /// each row asserting the outcome, what was touched, and for the row that stops + /// the message the preview prints about it. It runs the real resolution against + /// a served index, so a preview that fetched `uv` would show up as an acquire + /// or a tools folder, and a real install that stopped fetching it would show up + /// as a plan with no versions. + #[cfg(unix)] + #[test] + fn a_preview_never_acquires_uv_and_a_real_resolution_still_does() { + use SourceLayout::{Canonical, Next}; + use UvAccess::{Acquire, PresentOnly}; + + // ROCm 10, a preview, no uv present: the one row that stops. It asks + // whether one is present and touches nothing, so no `uv` ran, none was + // acquired and no tools folder was made. + let run = resolve_through_policy(Next, PresentOnly, false, false); + let Ok(PipRuntimePlan::NeedsUv(unresolved)) = &run.plan else { + panic!("expected a plan that stopped before uv, got {:?}", run.plan); + }; + assert_eq!( + ( + run.looked, + run.acquired, + run.uv_runs.as_str(), + run.tools_folder + ), + (1, 0, "", false), + "a preview with no uv present must acquire nothing and run nothing" + ); + // What it knows without `uv`: the ROCm version, the pinned torchaudio and + // the requirement lines `uv` would be given. + assert_eq!(unresolved.rocm_version, "10.0.0"); + assert_eq!(unresolved.torchaudio_pin.version, "2.9.1+rocm10.0.0"); + assert_eq!( + unresolved.version_resolution_specs, + [ + "rocm[libraries,device-gfx1200]==10.0.0", + "torch[device-gfx1200]", + "torchvision[device-gfx1200]", + "torchaudio==2.9.1+rocm10.0.0", + ] + ); + // And the message says what it did not do, in the same row. + let text = preview_text_with_prefix(unresolved, None); + for line in [ + " runtime_key: not resolved by this preview", + " package_specs: not resolved by this preview", + " command: not resolved by this preview", + " manifest: not resolved by this preview", + ] { + assert!( + text.lines().any(|l| l == line), + "missing `{line}` in:\n{text}" + ); + } + assert!( + text.contains("resolution_note: no uv is installed"), + "the preview must say why:\n{text}" + ); + assert!( + !text.contains("2.10.0+rocm10.0.0") && !text.contains("0.25.0+rocm10.0.0"), + "no version only uv could have chosen may be printed:\n{text}" + ); + + // ROCm 10, a preview, a uv present: it runs, once, and the plan is the + // one a real install reaches. Nothing is acquired. + let run = resolve_through_policy(Next, PresentOnly, true, false); + let stack = stack_of(run.plan.as_ref().expect("resolves with the present uv")); + assert_eq!( + (stack.torch.as_str(), stack.compatibility_key.as_str()), + ("2.10.0+rocm10.0.0", "10.0.0") + ); + assert_eq!( + (run.looked, run.acquired, run.tools_folder), + (1, 0, false), + "a preview with a uv present must not acquire another" + ); + assert_eq!(run.uv_runs.lines().count(), 1, "{}", run.uv_runs); + assert!(run.uv_runs.starts_with("pip compile "), "{}", run.uv_runs); + + // ROCm 10, a real install or an update: the managed uv is acquired, once, + // whether or not one is present (acquiring is what finds an installed + // one), and never looked for first. This is what must not change. + for uv_present in [false, true] { + let run = resolve_through_policy(Next, Acquire, uv_present, true); + let stack = stack_of(run.plan.as_ref().expect("resolves with the acquired uv")); + assert_eq!(stack.torch, "2.10.0+rocm10.0.0"); + assert_eq!( + (run.looked, run.acquired), + (0, 1), + "an install acquires uv, with one present: {uv_present}" + ); + assert_eq!(run.uv_runs.lines().count(), 1, "{}", run.uv_runs); + } + // A failed acquisition fails the resolution with the context it always had, + // and no plan stands in for it. + let run = resolve_through_policy(Next, Acquire, false, false); + let error = format!( + "{:#}", + run.plan.expect_err("an install cannot go without uv") + ); + assert!( + error.contains("failed to acquire uv for ROCm X metadata resolution: the managed uv could not be downloaded"), + "{error}" + ); + assert_eq!((run.looked, run.acquired), (0, 1)); + + // The canonical layout needs no uv for either access, present or not: the + // versions are read off the simple indexes, and nothing is looked for, + // acquired or run. + for access in [Acquire, PresentOnly] { + for uv_present in [false, true] { + let run = resolve_through_policy(Canonical, access, uv_present, true); + let stack = stack_of(run.plan.as_ref().expect("resolves from the indexes")); + assert_eq!( + (stack.rocm.as_str(), stack.torch.as_str()), + ("7.10.0", "2.9.0+rocm7.10.0") + ); + assert_eq!( + ( + run.looked, + run.acquired, + run.uv_runs.as_str(), + run.tools_folder + ), + (0, 0, "", false), + "{access:?}, uv present: {uv_present}" + ); + } + } + } + + // ---- the plan a preview without uv prints -------------------------------- + // + // Built by hand rather than from a served index, so that these run on every + // platform, the Windows CI job included: the wording and the shape of the plan + // do not depend on a stand-in `uv`. + + const PLAN_INDEX: &str = "http://index.test/next"; + + fn rocm10_specs() -> Vec { + published_pip_requirements("10.0.0", "2.9.1+rocm10.0.0", "gfx1200", false) + } + + fn rocm10_pin() -> TorchaudioPin { + torchaudio_pin(&["2.9.1+rocm10.0.0".to_owned()], "10.0.0", None, PLAN_INDEX) + .expect("the one published torchaudio is pinned") + } + + /// What a ROCm 10 resolution knows before `uv` has run. + fn unresolved_runtime() -> UnresolvedPipRuntime { + UnresolvedPipRuntime { + family: "gfx1200".to_owned(), + family_source: "manifest".to_owned(), + index_url: PLAN_INDEX.to_owned(), + layout: SourceLayout::Next, + rocm_version: "10.0.0".to_owned(), + newest_repo_version: None, + version_resolution_specs: rocm10_specs(), + torchaudio_pin: rocm10_pin(), + device_target: AggregateDeviceTarget::Exact("gfx1200".to_owned()), + published_device_targets: vec!["gfx1200".to_owned()], + } + } + + /// The same resolution once `uv` has chosen the versions. + fn resolved_runtime() -> PipRuntimeResolution { + let unresolved = unresolved_runtime(); + PipRuntimeResolution { + family: unresolved.family, + family_source: unresolved.family_source, + index_url: unresolved.index_url, + layout: unresolved.layout, + latest_version: "10.0.0".to_owned(), + newest_repo_version: None, + version_resolution_specs: Some(unresolved.version_resolution_specs), + torchaudio_pin: Some(unresolved.torchaudio_pin), + package_versions: TheRockPipPackageVersions { + rocm: "10.0.0".to_owned(), + torch: "2.10.0+rocm10.0.0".to_owned(), + torchvision: "0.25.0+rocm10.0.0".to_owned(), + torchaudio: "2.9.1+rocm10.0.0".to_owned(), + compatibility_key: "10.0.0".to_owned(), + }, + device_target: unresolved.device_target, + published_device_targets: unresolved.published_device_targets, + } + } + + /// The Python a clean host would prepare, and the wheel tags that follow from it. + fn clean_host_python() -> ( + super::super::super::python::PlannedPython, + WheelCompatibility, + ) { + use super::super::super::python::{PlannedPython, TagCheck, python_requirement}; + ( + PlannedPython::WouldPrepareManaged { + version: "3.14".to_owned(), + requirement: python_requirement(SourceLayout::Next), + tag_check: TagCheck::Checked, + }, + WheelCompatibility { + python_tag: "cp314".to_owned(), + platform_tags: vec!["linux_x86_64".to_owned(), "any".to_owned()], + }, + ) + } + + fn preview_text_with_prefix( + unresolved: &UnresolvedPipRuntime, + prefix: Option<&Path>, + ) -> String { + let (python, wheel_compatibility) = clean_host_python(); + super::super::super::wheel_plan::unresolved_wheel_preview( + &super::super::super::wheel_plan::UnresolvedPreview { + channel: TheRockChannel::Release, + version_selector: None, + unresolved, + device_target_override: None, + prefix, + host_version_newer: None, + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ) + } + + /// The whole text, so that no line can be added, dropped or reworded without + /// this test saying so. Everything made from the torch and torchvision + /// versions reads `not resolved by this preview`, and `resolution_note` says + /// why, in the voice of `python_note` beside it. + #[test] + fn the_plan_a_preview_without_uv_prints_is_exactly_this() { + let unresolved = unresolved_runtime(); + let text = preview_text_with_prefix(&unresolved, None); + let generation = unresolved.layout.generation(); + let pin = unresolved.torchaudio_pin.describe(); + let expected = format!( + "\ +sdk install + summary: rocm-cli will install the ROCm SDK and matching PyTorch packages for this Python and operating system + channel: release + canonical_source: {PLAN_INDEX} + selected_rocm_version: 10.0.0 + build_date: not encoded in stable version + source_layout_generation: {generation} + format: wheel + target_family: gfx1200 + target_family_source: manifest + device_target: gfx1200 + index_url: {PLAN_INDEX} + latest_compatible_version: 10.0.0 + compatibility_key: 10.0.0 + target: not resolved by this preview + runtime_key: not resolved by this preview + python_launcher: not prepared by this preview + python_source: managed + python_note: no usable Python was found; a real install would use uv to install managed Python 3.14 and save its path in the configuration, which this preview did not do + resolution_note: no uv is installed, so a real install would fetch it and run `uv pip compile` to choose the torch and torchvision versions and to check that the published packages install together, which this preview did not do; the package specs, runtime key, install folder, manifest and command depend on those versions and are not resolved here + python_wheel_tag: cp314 + platform_wheel_tags: linux_x86_64,any + version_resolution_specs: rocm[libraries,device-gfx1200]==10.0.0 torch[device-gfx1200] torchvision[device-gfx1200] torchaudio==2.9.1+rocm10.0.0 + torchaudio_selection: {pin} + package_specs: not resolved by this preview + package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction + mode: dry-run + command: not resolved by this preview + activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process + manifest: not resolved by this preview +" + ); + assert_eq!(text, expected); + } + + /// A `--prefix` does not depend on the versions, so it is the plan's target + /// even though the runtime key is not known; the key is what would otherwise + /// name the folder. + #[test] + fn an_explicit_prefix_is_the_target_of_a_preview_without_uv() { + let (root, _) = super::super::super::test_support::test_paths("uv-access-prefix"); + std::fs::create_dir_all(&root).expect("create the prefix parent"); + let prefix = root.join("my runtime"); + let text = preview_text_with_prefix(&unresolved_runtime(), Some(&prefix)); + let resolved = rocm_core::resolve_path_through_symlinks(&prefix); + let _ = std::fs::remove_dir_all(&root); + assert!( + text.lines() + .any(|line| line == format!(" target: {}", resolved.display())), + "{text}" + ); + assert!( + text.lines() + .any(|line| line == " runtime_key: not resolved by this preview"), + "{text}" + ); + } + + /// The keys a plan prints, in order. + fn plan_keys(text: &str) -> Vec<&str> { + text.lines() + .filter_map(|line| line.strip_prefix(" ")) + .filter_map(|line| line.split_once(": ").map(|(key, _)| key)) + .collect() + } + + /// One writer serves both plans, so a line the resolved plan gains cannot be + /// missing from the other: the two print the same keys in the same order, and + /// the unresolved one adds only `resolution_note`. + #[test] + fn a_resolved_and_an_unresolved_plan_print_the_same_keys() { + use super::super::super::wheel_plan::{ + PlanStack, ResolvedStack, WheelPlan, write_dry_run_tail, write_wheel_plan, + }; + + let unresolved = preview_text_with_prefix(&unresolved_runtime(), None); + let resolution = resolved_runtime(); + let composition = wheel_runtime_composition(&resolution, &resolution.device_target, false); + let runtime_key = super::super::super::wheel_runtime_key( + TheRockChannel::Release, + &resolution.latest_version, + &composition, + ); + let install_root = PathBuf::from("/runtimes").join(&runtime_key); + let manifest_path = PathBuf::from("/manifests").join(&runtime_key); + let (python, compatibility) = clean_host_python(); + let plan = WheelPlan { + channel: TheRockChannel::Release, + version_selector: None, + index_url: &resolution.index_url, + layout_generation: resolution.layout.generation(), + latest_version: &resolution.latest_version, + family: &resolution.family, + family_source: &resolution.family_source, + device_target: &resolution.device_target, + host_version_newer: None, + python: &python, + wheel_compatibility: &compatibility, + version_resolution_specs: resolution.version_resolution_specs.as_deref(), + torchaudio_pin: resolution.torchaudio_pin.as_ref(), + no_wheel_warning: None, + stack: PlanStack::Resolved(ResolvedStack { + compatibility_key: &resolution.package_versions.compatibility_key, + install_root: &install_root, + runtime_key: &runtime_key, + package_specs: &composition.package_specs, + manifest_path: &manifest_path, + }), + }; + let mut resolved = String::new(); + write_wheel_plan(&mut resolved, &plan); + write_dry_run_tail(&mut resolved, &plan); + + let mut expected = plan_keys(&resolved); + let at = expected + .iter() + .position(|key| *key == "python_note") + .expect("a python_note: this host would prepare Python") + + 1; + expected.insert(at, "resolution_note"); + assert_eq!(plan_keys(&unresolved), expected); + // And the resolved plan carries what the unresolved one cannot: the + // versions `uv` chose, in the package specs, and a real folder and key. + assert!( + resolved.contains("torch[device-gfx1200]==2.10.0+rocm10.0.0"), + "{resolved}" + ); + assert!(!resolved.contains("not resolved"), "{resolved}"); + assert!(!resolved.contains("resolution_note"), "{resolved}"); + } } diff --git a/apps/rocm/src/therock/wheel_plan.rs b/apps/rocm/src/therock/wheel_plan.rs index 6c6cfc125f..4deb719cd2 100644 --- a/apps/rocm/src/therock/wheel_plan.rs +++ b/apps/rocm/src/therock/wheel_plan.rs @@ -6,20 +6,31 @@ //! //! A preview prints it and stops; a real install prints it as the head of its //! own output. [`write_wheel_plan`] writes the lines both share, and -//! [`write_dry_run_tail`] the lines only a preview adds. Moved out of -//! `install_wheel_runtime` so that one writer serves every plan it prints. +//! [`write_dry_run_tail`] the lines only a preview adds. +//! +//! One writer serves a resolved plan and one that could not be resolved, so the +//! two cannot drift apart: a line the plan gains appears in both. They differ +//! only in the lines made from the package versions `uv pip compile` chooses on +//! the ROCm 10 layout (`package_specs`, `runtime_key`, `target`, `manifest` and +//! `command`). [`PlanStack::NeedsUv`] says those are not resolved, and why, +//! where [`PlanStack::Resolved`] carries their values. use super::python::{PlannedPython, WheelCompatibility}; -use super::resolve::{AggregateDeviceTarget, TorchaudioPin}; +use super::resolve::{AggregateDeviceTarget, TorchaudioPin, UnresolvedPipRuntime}; use super::source::{RuntimeVersionSelector, TheRockChannel}; use super::{ - render_canonical_provenance, runtime_version_display, venv_python_path, wheel_host_version_note, + no_wheel_warning_for, render_canonical_provenance, runtime_version_display, venv_python_path, + wheel_host_version_note, }; use crate::shell_quote::shell_quote; use rocm_core::{uv_pip_install_base, uv_venv_args}; use std::fmt::Write as _; use std::path::Path; +/// The value printed for a line that depends on the package versions `uv` has not +/// chosen. +pub(super) const NOT_RESOLVED: &str = "not resolved by this preview"; + /// The facts the plan prints, from a resolution and the Python decision. pub(super) struct WheelPlan<'a> { pub(super) channel: TheRockChannel, @@ -33,17 +44,45 @@ pub(super) struct WheelPlan<'a> { pub(super) device_target: &'a AggregateDeviceTarget, /// The host's ROCm when it is newer than `latest_version`. pub(super) host_version_newer: Option<&'a str>, - pub(super) compatibility_key: &'a str, - pub(super) install_root: &'a Path, - pub(super) runtime_key: &'a str, pub(super) python: &'a PlannedPython, pub(super) wheel_compatibility: &'a WheelCompatibility, /// The requirement lines version resolution asked `uv` about. `None` on the /// canonical layout, which resolves without `uv`. pub(super) version_resolution_specs: Option<&'a [String]>, pub(super) torchaudio_pin: Option<&'a TorchaudioPin>, - pub(super) package_specs: &'a [String], pub(super) no_wheel_warning: Option<&'a str>, + pub(super) stack: PlanStack<'a>, +} + +/// Everything in the plan that is made from the package versions. +pub(super) enum PlanStack<'a> { + /// The versions are chosen; these are what they produce. + Resolved(ResolvedStack<'a>), + /// A preview of the ROCm 10 layout with no `uv`: the versions were not chosen, + /// so nothing made from them can be printed. + NeedsUv { + /// The install folder when the user named one with `--prefix`: it does not + /// depend on the versions. Otherwise the runtime key names it. + explicit_install_root: Option<&'a Path>, + }, +} + +pub(super) struct ResolvedStack<'a> { + pub(super) compatibility_key: &'a str, + pub(super) install_root: &'a Path, + pub(super) runtime_key: &'a str, + pub(super) package_specs: &'a [String], + pub(super) manifest_path: &'a Path, +} + +/// The `compatibility_key` the install is recorded under is the `rocm` build every +/// package in the stack shares, so a plan that has not chosen the stack still +/// knows it: the `rocm` version. +fn compatibility_key<'a>(plan: &'a WheelPlan<'a>) -> &'a str { + match &plan.stack { + PlanStack::Resolved(stack) => stack.compatibility_key, + PlanStack::NeedsUv { .. } => plan.latest_version, + } } /// Writes the plan from `sdk install` to `package_policy` and the repository @@ -88,10 +127,27 @@ pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { let _ = writeln!( output, " compatibility_key: {}", - runtime_version_display(plan.compatibility_key) + runtime_version_display(compatibility_key(plan)) ); - let _ = writeln!(output, " target: {}", plan.install_root.display()); - let _ = writeln!(output, " runtime_key: {}", plan.runtime_key); + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " target: {}", stack.install_root.display()); + let _ = writeln!(output, " runtime_key: {}", stack.runtime_key); + } + PlanStack::NeedsUv { + explicit_install_root, + } => { + match explicit_install_root { + Some(root) => { + let _ = writeln!(output, " target: {}", root.display()); + } + None => { + let _ = writeln!(output, " target: {NOT_RESOLVED}"); + } + } + let _ = writeln!(output, " runtime_key: {NOT_RESOLVED}"); + } + } let _ = writeln!( output, " python_launcher: {}", @@ -101,6 +157,9 @@ pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { if let Some(note) = plan.python.note() { let _ = writeln!(output, " python_note: {note}"); } + if matches!(plan.stack, PlanStack::NeedsUv { .. }) { + let _ = writeln!(output, " resolution_note: {UNRESOLVED_NOTE}"); + } let _ = writeln!( output, " python_wheel_tag: {}", @@ -125,7 +184,14 @@ pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { if let Some(pin) = plan.torchaudio_pin { let _ = writeln!(output, " torchaudio_selection: {}", pin.describe()); } - let _ = writeln!(output, " package_specs: {}", plan.package_specs.join(" ")); + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " package_specs: {}", stack.package_specs.join(" ")); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " package_specs: {NOT_RESOLVED}"); + } + } let _ = writeln!( output, " package_policy: resolve the pinned target-complete rocm, torch, torchvision, and torchaudio plan from published package metadata, then install it in one uv transaction" @@ -135,41 +201,137 @@ pub(super) fn write_wheel_plan(output: &mut String, plan: &WheelPlan<'_>) { } } +/// What a preview says about the versions it did not choose. Written beside +/// `python_note`, in the same voice: what a real install would do, and that this +/// preview did not. +pub(super) const UNRESOLVED_NOTE: &str = "no uv is installed, so a real install would fetch it and run `uv pip compile` to choose the torch and torchvision versions and to check that the published packages install together, which this preview did not do; the package specs, runtime key, install folder, manifest and command depend on those versions and are not resolved here"; + +/// The progress line for the same decision as [`UNRESOLVED_NOTE`], in the voice of +/// the Python line (`... a real install would prepare Python X. This preview does +/// not.`). +pub(super) const UNRESOLVED_PROGRESS: &str = "No uv is installed; a real install would fetch it and resolve the torch and torchvision versions with `uv pip compile`. This preview does not."; + /// The lines a preview adds after [`write_wheel_plan`]: the commands a real install /// would run, and where it would record the runtime. -pub(super) fn write_dry_run_tail(output: &mut String, plan: &WheelPlan<'_>, manifest_path: &Path) { - let env_python = venv_python_path(plan.install_root); - let mut install_args = uv_pip_install_base(&env_python); - install_args.extend(["--index-url".to_owned(), plan.index_url.to_owned()]); - if matches!(plan.channel, TheRockChannel::Nightly) { - install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); - } - install_args.extend(plan.package_specs.iter().cloned()); - let venv_args = uv_venv_args(&plan.python.venv_python_arg(), plan.install_root); - // This line is offered as the command to run, so it has to be one a - // shell reads back as the argv it previews. The install root in it - // comes from a `--prefix` the local assistant can choose - // (`install_sdk_dry_run` is a model-callable tool), so a bare or - // double-quoted value here would be a model-chosen value in a line a - // user is invited to paste. - let venv_args_display = venv_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); - let install_args_display = install_args - .iter() - .map(|arg| shell_quote(arg)) - .collect::>() - .join(" "); +pub(super) fn write_dry_run_tail(output: &mut String, plan: &WheelPlan<'_>) { let _ = writeln!(output, " mode: dry-run"); - let _ = writeln!( - output, - " command: uv {venv_args_display} && uv {install_args_display}" - ); + match &plan.stack { + PlanStack::Resolved(stack) => { + let env_python = venv_python_path(stack.install_root); + let mut install_args = uv_pip_install_base(&env_python); + install_args.extend(["--index-url".to_owned(), plan.index_url.to_owned()]); + if matches!(plan.channel, TheRockChannel::Nightly) { + install_args.extend(["--prerelease".to_owned(), "allow".to_owned()]); + } + install_args.extend(stack.package_specs.iter().cloned()); + let venv_args = uv_venv_args(&plan.python.venv_python_arg(), stack.install_root); + // This line is offered as the command to run, so it has to be one a + // shell reads back as the argv it previews. The install root in it + // comes from a `--prefix` the local assistant can choose + // (`install_sdk_dry_run` is a model-callable tool), so a bare or + // double-quoted value here would be a model-chosen value in a line a + // user is invited to paste. + let venv_args_display = venv_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let install_args_display = install_args + .iter() + .map(|arg| shell_quote(arg)) + .collect::>() + .join(" "); + let _ = writeln!( + output, + " command: uv {venv_args_display} && uv {install_args_display}" + ); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " command: {NOT_RESOLVED}"); + } + } let _ = writeln!( output, " activation: use the managed venv Python; TheRock libraries are resolved from that venv by rocm_sdk.initialize_process" ); - let _ = writeln!(output, " manifest: {}", manifest_path.display()); + match &plan.stack { + PlanStack::Resolved(stack) => { + let _ = writeln!(output, " manifest: {}", stack.manifest_path.display()); + } + PlanStack::NeedsUv { .. } => { + let _ = writeln!(output, " manifest: {NOT_RESOLVED}"); + } + } +} + +/// What [`unresolved_wheel_preview`] is made from. +pub(super) struct UnresolvedPreview<'a> { + pub(super) channel: TheRockChannel, + pub(super) version_selector: Option<&'a RuntimeVersionSelector>, + pub(super) unresolved: &'a UnresolvedPipRuntime, + /// An update apply's recorded exact arch, resolved against the targets the + /// source publishes exactly as a resolved plan does. + pub(super) device_target_override: Option<&'a str>, + /// The `--prefix` the user named. It does not depend on the versions, so it is + /// the plan's `target`; without one the runtime key names the install folder + /// and the `target` is not resolved. + pub(super) prefix: Option<&'a Path>, + /// The host's ROCm when it is newer than the version the plan is for. + pub(super) host_version_newer: Option<&'a str>, + pub(super) python: &'a PlannedPython, + pub(super) wheel_compatibility: &'a WheelCompatibility, +} + +/// The whole plan for a preview of the ROCm 10 layout that stopped before +/// `uv pip compile`: what was chosen without `uv`, written by the same writer as +/// every other plan, with the lines made from the unchosen versions marked. +pub(super) fn unresolved_wheel_preview(preview: &UnresolvedPreview<'_>) -> String { + let UnresolvedPreview { + channel, + version_selector, + unresolved, + device_target_override, + prefix, + host_version_newer, + python, + wheel_compatibility, + } = *preview; + let device_target = device_target_override.map_or_else( + || unresolved.device_target.clone(), + |target| { + AggregateDeviceTarget::resolve( + Some(target), + &unresolved.family, + &unresolved.published_device_targets, + ) + }, + ); + let explicit_install_root = prefix.map(rocm_core::resolve_path_through_symlinks); + let no_wheel_warning = no_wheel_warning_for( + unresolved.newest_repo_version.as_deref(), + &unresolved.rocm_version, + ); + let plan = WheelPlan { + channel, + version_selector, + index_url: &unresolved.index_url, + layout_generation: unresolved.layout.generation(), + latest_version: &unresolved.rocm_version, + family: &unresolved.family, + family_source: &unresolved.family_source, + device_target: &device_target, + host_version_newer, + python, + wheel_compatibility, + version_resolution_specs: Some(&unresolved.version_resolution_specs), + torchaudio_pin: Some(&unresolved.torchaudio_pin), + no_wheel_warning: no_wheel_warning.as_deref(), + stack: PlanStack::NeedsUv { + explicit_install_root: explicit_install_root.as_deref(), + }, + }; + let mut output = String::new(); + write_wheel_plan(&mut output, &plan); + write_dry_run_tail(&mut output, &plan); + output } diff --git a/docs/architecture.md b/docs/architecture.md index 0fa6e056ec..d05957160b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,9 +111,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/probe.rs`](../apps/rocm/src/therock/probe.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/process.rs` and `therock/python.rs` (split out of `therock.rs`): asking an installed runtime about itself through its own interpreter. `probe_rocm_sdk_runtime` runs `ROCM_SDK_PROBE_SCRIPT` and parses what `rocm_sdk` reports into the `RocmSdkPythonProbe` that [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) owns and validates; `probe_runtime_devices` runs `RUNTIME_DEVICE_PROBE_SCRIPT` to count the devices the runtime's torch can open and whether one can run a kernel (`RuntimeDeviceProbe`); `probe_torch_alignment` reads the installed torch and the one an engine pins from distribution metadata without importing torch (`TorchAlignmentProbe`); and `split_local_version`, `requirement_pinned_version` and `install_pinned_package` are the torch-pin helpers [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) uses to put the SDK's torch back after an engine install. The `pub(crate)` items are re-exported by `therock.rs` for [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs); `probe_rocm_sdk_runtime_for_target`, which `install_wheel_runtime` calls with the device payload it just chose, is `pub(super)`. - [`therock/resolve.rs`](../apps/rocm/src/therock/resolve.rs) — a child of `therock.rs` that holds no logic (split out of `therock.rs`): only the `mod` declarations for the artifact-resolution modules under `therock/resolve/` and the `pub(super) use` lines through which `therock.rs` reads them. It is the parent the `resolve/` files hang from, so a module there is `pub(in crate::therock)` for the items `therock.rs` reaches and private otherwise. - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. -- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. +- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. That compile is the one step that may fetch `uv`, so who may is decided in one pure function: `plan_version_resolution` takes the layout, a `UvAccess` and whether a `uv` is present, and answers scrape the indexes (canonical), compile with a `UvChoice` (acquire the managed `uv`, or use the present one) or skip. `resolve_pip_runtime` and `resolve_pip_runtime_with_timeout` (a real install and `rocm update`) are always `UvAccess::Acquire`, which never skips and never asks whether a `uv` is present; `preview_pip_runtime` is `UvAccess::PresentOnly`, which uses a present `uv` (`rocm_core::find_uv_binary`) and otherwise returns `PipRuntimePlan::NeedsUv` holding an `UnresolvedPipRuntime`: what was chosen without `uv` (the `rocm` version, the torchaudio pin, the requirements `uv` would be given) and no package versions, so nothing made from them can be built from it. The real `find_uv_binary` and `ensure_uv_binary` reach the resolution through a `UvPolicy` so that a test can count the calls. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. -- [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) — mechanical relocation out of `install_wheel_runtime` in `therock.rs`, a leaf child of `therock.rs` that builds on `therock/python.rs` and `therock/resolve.rs` for the Python decision and the resolution it prints: the `sdk install` plan a wheel install prints, key by key. `WheelPlan` carries the facts, `write_wheel_plan` writes the lines a preview and a real install share, and `write_dry_run_tail` the lines only a preview adds (the `uv` commands it would run and the manifest path). One writer serves every plan, so a line the plan gains appears wherever it is printed. `therock.rs` reads it through `pub(super)` items; it reads the provenance, host-note and `venv_python_path` helpers back from `therock.rs`. +- [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) — full domain extraction, a leaf child of `therock.rs` (split out of `install_wheel_runtime`) that builds on `therock/python.rs` and `therock/resolve.rs` for the Python decision and the resolution it prints: the `sdk install` plan a wheel install prints, key by key. `WheelPlan` carries the facts, `write_wheel_plan` writes the lines a preview and a real install share, and `write_dry_run_tail` the lines only a preview adds (the `uv` commands it would run and the manifest path). `PlanStack` is the one place the plan differs: `Resolved` carries the package specs, runtime key, install folder and manifest path, and `NeedsUv` (a ROCm 10 preview on a host with no `uv`, see `UnresolvedPipRuntime` in `therock/resolve/wheel.rs`) prints those as `not resolved by this preview` and adds the `resolution_note:` line, so one writer serves both and a line the plan gains appears in both. `unresolved_wheel_preview` renders the second kind whole. `therock.rs` reads it through `pub(super)` items; it reads the provenance, host-note and `venv_python_path` helpers back from `therock.rs`. - [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. - [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK` and the background-helper fixtures `runtime_state` and `set_helper_facts`. - [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_serve_command_safety`, the four rules a model-proposed `serve` must satisfy (no public bind, no CPU device, a loopback host, `--managed` without `--foreground`), and `validate_chat_rocm_command_safety`, which applies them to a `serve` argv and also refuses a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_serve_command_safety` and `validate_chat_rocm_command_safety` here instead of through `chat`, and keeps no copy of the serve rules. `chat.rs`'s child modules call into it and `chat.rs` re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index a698912fd5..6f365fd79c 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -601,3 +601,31 @@ Feature: Runtime configuration When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 Then the preview requests the gfx1200 device extras And the preview resolved the versions with the uv that was present + + # A preview is a promise that nothing on the machine changes, and runtime-23 held + # the Python half of it. The other half is `uv`: a preview of a ROCm 10 or newer + # wheel used to go and fetch the managed `uv` (creating the data folder's `tools` + # folder) because `uv pip compile` is the one step that knows the torch and + # torchvision versions. On a host with no `uv` the preview now runs nothing and + # downloads nothing. It prints what it can know without `uv` (the ROCm version, + # the torchaudio pin, the requirements `uv` would be asked about), says in a + # `resolution_note:` line and a progress line what it did not do, and marks the + # lines made from the missing versions as not resolved rather than printing a + # guess. A real install and `rocm update` still fetch `uv` as they always did. + # + # The Then checks the message and the state together: with no `uv` to log, the + # evidence is the folders. No managed `tools` folder, no tool archive in the + # scenario's own cache, no saved configuration, and no torch version anywhere in + # the output, so the unresolved lines are not covering for a guess. runtime-27 + # is the other polarity: with a `uv` present the compile still runs. + # + # Linux only, for the same empty-`PATH` host as runtime-23. The decision is + # unit-tested for every combination of layout, who may fetch `uv` and whether one + # is present in `apps/rocm/src/therock/resolve/wheel.rs`, and the plan's + # wording in `apps/rocm/src/therock/wheel_plan.rs`, on every platform. + @id:runtime-install-preview-of-rocm-10-fetches-no-uv @requires-os:linux + Scenario: runtime-28 - Previewing a ROCm 10 SDK install on a host with no uv fetches none and says what it did not resolve + Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a host with no uv and no usable Python + When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 + Then the preview says it did not resolve the ROCm 10 package versions, as it says diff --git a/tests/e2e-cucumber/features/therock_next_generation.feature b/tests/e2e-cucumber/features/therock_next_generation.feature index 35c683eccf..58ce4a24c4 100644 --- a/tests/e2e-cucumber/features/therock_next_generation.feature +++ b/tests/e2e-cucumber/features/therock_next_generation.feature @@ -11,6 +11,13 @@ Feature: TheRock "next" ROCm 10 install layout # Scenarios 01-05 point source overrides at loopback fixtures through the # explicit trust gate. Scenario 06 proves the same overrides are inert when # the trust gate is absent; scenario 07 supplies the live GPU evidence. + # + # A ROCm 10 preview has a real `uv pip compile` choose the torch and torchvision + # versions, and a preview never fetches `uv` itself. The scenarios below that + # print resolved package specs (02, 10, 11 and 13) therefore start from an + # installed `uv`, which is the precondition they always had when the preview + # fetched one for them. runtime-27 and runtime-28 in `runtime_setup.feature` pin + # the preview's behaviour with a `uv` present and with none. # The regression this pins: a "next-first" dispatch that probes the ROCm 10 # sources before the canonical ones. An unpinned release install must never @@ -30,6 +37,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-02-wheel-pins-raw-device-extras Scenario: therock-next-02 - A pinned ROCm 10 wheel install resolves the next index with raw device extras Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 Then the preview resolves the ROCm 10 pip index And the preview reports the next source layout generation @@ -195,6 +203,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-09-wheel-devel-adds-the-toolchain Scenario: therock-next-10 - A pinned ROCm 10 wheel install adds the toolchain when asked Given a canonical release pip index fixture and a ROCm 10 pip index fixture + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 with the toolchain Then the preview resolves the ROCm 10 pip index And the preview requests the gfx1200 device extras with the toolchain @@ -210,6 +219,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-11-older-pin-keeps-one-rocm-build Scenario: therock-next-11 - A ROCm 10 wheel install pinned below the newest build installs only that build Given a ROCm 10 pip index fixture that also publishes a newer ROCm 10 build + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0 Then the preview resolves the ROCm 10 pip index And every package in the preview is built for ROCm 10.0.0 @@ -235,6 +245,7 @@ Feature: TheRock "next" ROCm 10 install layout @id:therock-next-13-verified-torchaudio-over-newer-rebuild Scenario: therock-next-13 - A ROCm 10 wheel install takes the hardware-verified torchaudio over a newer rebuild Given a ROCm 10 pip index fixture where several torchaudio rebuilds share one ROCm label + And a uv is already installed When the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.1.0 Then the preview installs the hardware-verified torchaudio for ROCm 10.1.0, not the newer rebuild diff --git a/tests/e2e-cucumber/tests/e2e/therock_steps.rs b/tests/e2e-cucumber/tests/e2e/therock_steps.rs index 61c4999e9c..9a3ee87d27 100644 --- a/tests/e2e-cucumber/tests/e2e/therock_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/therock_steps.rs @@ -283,6 +283,22 @@ async fn host_with_no_python_and_recording_uv(world: &mut E2eWorld) { )); } +/// The same host with no `uv` anywhere: none named by `ROCM_CLI_UV_BINARY` and no +/// managed copy under the scenario's data folder, so a CLI that needs one would +/// have to download it. Nothing here can observe a `uv` run, since there is none to +/// run; what it can observe is the state a download leaves, which is why the +/// scenario's data and cache folders are the evidence. +#[given("a host with no uv and no usable Python")] +async fn host_with_no_uv_and_no_python(world: &mut E2eWorld) { + assert!( + std::env::var_os(rocm_core::uv::UV_BINARY_ENV).is_none(), + "{} is set in the environment running the suite, which would put a uv on this host; \ + unset it", + rocm_core::uv::UV_BINARY_ENV + ); + host_without_python(world); +} + /// What the hosts these steps build share: `PATH` is one empty folder, so no Python is found /// and none has been saved, and the CLI has a cache folder of its own. Returns /// the scenario root. @@ -873,6 +889,18 @@ async fn preview_unpinned_wheel_install(world: &mut E2eWorld) { ); } +/// A `uv` the preview can use without fetching one. The resolution these +/// scenarios assert on is a real `uv pip compile` against the loopback fixtures, +/// so the stand-in `uv` of runtime-27 would not do; this installs the managed one +/// the way a real install would have (from the network, into the scenario's own +/// data folder, or, when `ROCM_CLI_UV_BINARY` names one, that one). +#[given("a uv is already installed")] +async fn a_uv_is_already_installed(world: &mut E2eWorld) { + let paths = world.app_paths(); + rocm_core::ensure_uv_binary(&paths) + .unwrap_or_else(|error| panic!("failed to install uv for the scenario: {error:#}")); +} + #[when("the user previews a wheel SDK install for arch gfx1200 pinned to ROCm 10.0.0")] async fn preview_pinned_wheel_install(world: &mut E2eWorld) { preview_ok( @@ -1164,10 +1192,11 @@ async fn preview_prepared_no_python(world: &mut E2eWorld) { /// The state half of the claim: the recording `uv` was never run (the evidence /// that nothing was prepared), no managed tools folder, no saved configuration, -/// and, as a backstop only, no tool archive in the scenario's cache. Every -/// scenario that calls this previews the canonical layout (no ROCm 10 pin), the -/// only layout whose preview needs no `uv`; a ROCm 10 preview can legitimately -/// create the managed tools folder to resolve package versions. +/// and, as a backstop only, no tool archive in the scenario's cache. It holds for +/// both layouts, because a preview never fetches `uv`: the canonical layout needs +/// none, and the ROCm 10 layout runs one only when it is already present (see +/// runtime-27 and runtime-28). A scenario that gives the preview a `uv` and expects +/// it to run must not call this. fn assert_no_python_prepared(world: &E2eWorld) { let paths = world.app_paths(); let scenario_root = root(world).to_path_buf(); @@ -1352,6 +1381,68 @@ async fn preview_resolved_with_present_uv(world: &mut E2eWorld) { } } +/// What a preview of the ROCm 10 layout says when no `uv` is there to run, and the +/// state it left. The claim and the filesystem are checked together: the note alone +/// would pass with a preview that also downloaded `uv`, and the empty folders alone +/// would pass if the preview had stopped before it got to the question. +#[then("the preview says it did not resolve the ROCm 10 package versions, as it says")] +async fn preview_did_not_resolve_without_uv(world: &mut E2eWorld) { + assert_contains( + world, + "No uv is installed; a real install would fetch it and resolve the torch and torchvision \ + versions with `uv pip compile`. This preview does not.", + "the progress line about what the preview skipped", + ); + assert_contains( + world, + "resolution_note: no uv is installed, so a real install would fetch it and run `uv pip \ + compile` to choose the torch and torchvision versions and to check that the published \ + packages install together, which this preview did not do", + "the note about what the preview skipped", + ); + // Everything made from the versions `uv` would choose is marked, not guessed. + for key in [ + "target", + "runtime_key", + "package_specs", + "command", + "manifest", + ] { + assert_contains( + world, + &format!("\n {key}: not resolved by this preview\n"), + "a line made from the unchosen versions", + ); + } + // What needs no `uv` is still printed, from the same decisions a real install + // takes: the exact device payload, the requirements `uv` would be asked about, + // and the pinned torchaudio. + assert_contains( + world, + &format!("device_target: {RAW_ARCH}"), + "the device target", + ); + assert_version_resolution_extras(world, false); + assert_contains( + world, + &format!("selected_rocm_version: {NEXT_ROCM_VERSION}"), + "the ROCm version", + ); + assert_contains( + world, + &format!("torchaudio_selection: torchaudio {NEXT_TORCHAUDIO_VERSION}, "), + "the torchaudio pin", + ); + for chosen_by_uv in [NEXT_TORCH_VERSION, NEXT_TORCHVISION_VERSION] { + assert!( + !stdout(world).contains(chosen_by_uv), + "the preview named {chosen_by_uv}, which only `uv pip compile` could have chosen:\n{}", + stdout(world) + ); + } + assert_no_python_prepared(world); +} + #[then("the preview resolves the canonical release pip index")] async fn preview_resolves_canonical_pip_index(world: &mut E2eWorld) { let base = current_pip_base(world); From 44fd88db213e936e4fa447bb9491ff4e9bf283b7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:51:34 +0000 Subject: [PATCH 05/11] refactor(install): move the uv access decision out of the wheel resolver The previous commit took therock/resolve/wheel.rs past the 1500 production-line limit. Move the part that decides who may fetch uv (UvAccess, UvPolicy, VersionResolution, UvChoice, plan_version_resolution and uv_for_compile) and its state-matrix tests into therock/resolve/uv_access.rs. It builds only on SourceLayout, so wheel.rs keeps acting on the answer and nothing else changes: the code and the tests are moved as they were, with the items widened to pub(super) so wheel.rs can name them. Signed-off-by: Jussi Elo --- apps/rocm/src/therock/resolve.rs | 4 +- apps/rocm/src/therock/resolve/uv_access.rs | 215 +++++++++++++++++++++ apps/rocm/src/therock/resolve/wheel.rs | 196 +------------------ docs/architecture.md | 3 +- 4 files changed, 225 insertions(+), 193 deletions(-) create mode 100644 apps/rocm/src/therock/resolve/uv_access.rs diff --git a/apps/rocm/src/therock/resolve.rs b/apps/rocm/src/therock/resolve.rs index 05e7f2a7d5..6e1aca6340 100644 --- a/apps/rocm/src/therock/resolve.rs +++ b/apps/rocm/src/therock/resolve.rs @@ -5,9 +5,11 @@ //! How a request is turned into the artifact an install fetches. Only module //! declarations and re-exports live here, no logic: `tarball` resolves the SDK //! tarball from the catalog's index, `wheel` resolves the wheel stack from the -//! wheel indexes, and the rest of `therock` reads the names below. +//! wheel indexes (with `uv_access` deciding who may fetch `uv` while it does), +//! and the rest of `therock` reads the names below. mod tarball; +mod uv_access; mod wheel; pub(super) use self::tarball::{resolve_tarball_artifact, resolve_tarball_artifact_with_timeout}; diff --git a/apps/rocm/src/therock/resolve/uv_access.rs b/apps/rocm/src/therock/resolve/uv_access.rs new file mode 100644 index 0000000000..3ad8dbdb42 --- /dev/null +++ b/apps/rocm/src/therock/resolve/uv_access.rs @@ -0,0 +1,215 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Who may fetch `uv` while a wheel stack is resolved. +//! +//! On the ROCm 10 layout `uv pip compile` is the one step of resolution that can +//! download the managed `uv` and create the data folder's `tools` folder. A real +//! install and an update ([`UvAccess::Acquire`]) take it as they always did; a +//! preview ([`UvAccess::PresentOnly`]) runs a `uv` that is already there and +//! otherwise skips the step. [`plan_version_resolution`] decides which, as a pure +//! function of the layout, the access and whether a `uv` is present, so that every +//! combination is tested without a network. `wheel.rs` acts on the answer. + +use super::super::source::SourceLayout; +use anyhow::{Context, Result}; +use std::path::PathBuf; + +/// Whether a resolution may fetch `uv` to choose the ROCm 10 package versions. +/// +/// Only the ROCm 10 (`next`) layout needs `uv`: it has `uv pip compile` pick the +/// `torch` and `torchvision` builds. That is the one step of resolution that can +/// download the managed `uv` and create the data folder's `tools` folder. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum UvAccess { + /// A real install or an update: take the managed `uv`, downloading it if no + /// copy is installed yet. This is what resolution has always done. + Acquire, + /// A preview: use a `uv` that is already there and never fetch one. + PresentOnly, +} + +/// What a resolution may do about `uv`: how far it may go ([`UvAccess`]) and the two +/// things it can ask for. The real ones, `find_uv_binary` and `ensure_uv_binary`, +/// are built where a resolution starts; they are fields so that a test can count +/// the calls instead of reaching the network. +pub(super) struct UvPolicy<'a> { + pub(super) access: UvAccess, + /// A `uv` that is already there, without acquiring one. + pub(super) find_present: &'a dyn Fn() -> Option, + /// The managed `uv`, downloading it when none is installed. + pub(super) acquire: &'a dyn Fn() -> Result, +} + +/// How a resolution chooses the package versions, given [`UvAccess`] and whether a +/// `uv` is present. A pure function of those inputs: see [`plan_version_resolution`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum VersionResolution { + /// The canonical layout: read each package's simple index. No `uv`. + ScrapeIndexes, + /// The ROCm 10 layout: run `uv pip compile` with this `uv`. + Compile(UvChoice), + /// The ROCm 10 layout with no `uv` to run and none to fetch: stop with what is + /// known without one, and leave the versions `uv` would choose unresolved. + Skip, +} + +/// Which `uv` the compile step runs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum UvChoice { + /// Ask `ensure_uv_binary`, which downloads the managed `uv` when none is installed. + Acquire, + /// A `uv` that was already there. + Existing(PathBuf), +} + +/// Decide how the versions are chosen, without doing it. +/// +/// `find_present` looks for a `uv` that is already there. It runs only when the +/// answer depends on it: never for the canonical layout, which needs no `uv`, and +/// never for [`UvAccess::Acquire`], whose `ensure_uv_binary` does its own looking. +/// So a real install and an update decide, and call, exactly what they did before a +/// preview could skip the step, and no answer for them is [`VersionResolution::Skip`]. +pub(super) fn plan_version_resolution( + layout: SourceLayout, + access: UvAccess, + find_present: impl FnOnce() -> Option, +) -> VersionResolution { + match (layout, access) { + (SourceLayout::Canonical, _) => VersionResolution::ScrapeIndexes, + (SourceLayout::Next, UvAccess::Acquire) => VersionResolution::Compile(UvChoice::Acquire), + (SourceLayout::Next, UvAccess::PresentOnly) => find_present() + .map_or(VersionResolution::Skip, |uv| { + VersionResolution::Compile(UvChoice::Existing(uv)) + }), + } +} + +/// The `uv` the compile step runs: the one already there, or the managed `uv` that +/// `acquire` (`ensure_uv_binary`) fetches when none is installed. +pub(super) fn uv_for_compile( + choice: UvChoice, + acquire: impl FnOnce() -> Result, +) -> Result { + match choice { + UvChoice::Existing(uv) => Ok(uv), + UvChoice::Acquire => { + acquire().context("failed to acquire uv for ROCm X metadata resolution") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::bail; + + /// Asks [`plan_version_resolution`] for one cell of the matrix and reports + /// the answer together with how many times a present `uv` was looked for. + fn decide( + layout: SourceLayout, + access: UvAccess, + uv_present: bool, + ) -> (VersionResolution, usize) { + let looked = std::cell::Cell::new(0); + let decision = plan_version_resolution(layout, access, || { + looked.set(looked.get() + 1); + uv_present.then(|| PathBuf::from("/opt/uv")) + }); + (decision, looked.get()) + } + + /// The state matrix of the decision: both layouts, both accesses, a `uv` + /// present or not. Only the ROCm 10 layout needs `uv`; a real install and an + /// update (`Acquire`) always go on to acquire it and never ask whether one is + /// present, because `ensure_uv_binary` does its own looking; a preview + /// (`PresentOnly`) asks, and with none present stops. `Skip` therefore exists + /// for exactly one row. + #[test] + fn the_version_step_is_decided_by_layout_access_and_a_present_uv_alone() { + use SourceLayout::{Canonical, Next}; + use UvAccess::{Acquire, PresentOnly}; + let existing = || VersionResolution::Compile(UvChoice::Existing(PathBuf::from("/opt/uv"))); + let rows = [ + // (layout, access, uv present, decision, times a present uv was looked for) + ( + Canonical, + Acquire, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + Acquire, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + false, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Canonical, + PresentOnly, + true, + VersionResolution::ScrapeIndexes, + 0, + ), + ( + Next, + Acquire, + false, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + ( + Next, + Acquire, + true, + VersionResolution::Compile(UvChoice::Acquire), + 0, + ), + (Next, PresentOnly, false, VersionResolution::Skip, 1), + (Next, PresentOnly, true, existing(), 1), + ]; + for (layout, access, uv_present, expected, looked) in rows { + let (decision, calls) = decide(layout, access, uv_present); + let row = format!("{layout:?}, {access:?}, uv present: {uv_present}"); + assert_eq!(decision, expected, "decision for {row}"); + assert_eq!(calls, looked, "times a present uv was looked for in {row}"); + } + } + + /// The compile step runs the `uv` it was given, and asks for one only on + /// `Acquire`, with the context a failed fetch has always carried. + #[test] + fn the_compile_step_acquires_uv_only_when_told_to() { + let acquired = std::cell::Cell::new(0); + let acquire = || { + acquired.set(acquired.get() + 1); + Ok(PathBuf::from("/managed/uv")) + }; + assert_eq!( + uv_for_compile(UvChoice::Acquire, acquire).unwrap(), + PathBuf::from("/managed/uv") + ); + assert_eq!(acquired.get(), 1); + assert_eq!( + uv_for_compile(UvChoice::Existing(PathBuf::from("/opt/uv")), acquire).unwrap(), + PathBuf::from("/opt/uv") + ); + assert_eq!(acquired.get(), 1, "a present uv is not acquired again"); + + let error = uv_for_compile(UvChoice::Acquire, || bail!("no network")).unwrap_err(); + assert_eq!( + format!("{error:#}"), + "failed to acquire uv for ROCm X metadata resolution: no network" + ); + } +} diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index 03b2ec15f4..b8d6cb33d0 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -19,7 +19,7 @@ //! `uv`: a real install and an update (`resolve_pip_runtime`) acquire it, and a //! preview (`preview_pip_runtime`) runs one that is already present and //! otherwise stops with an `UnresolvedPipRuntime`, which is decided in -//! `plan_version_resolution`. `wheel_runtime_composition` turns a +//! `plan_version_resolution` (in `uv_access`). `wheel_runtime_composition` turns a //! resolution into the exact package specs (`therock_pip_package_specs`) an //! install gives `uv` and records in the manifest, and //! `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` @@ -49,6 +49,9 @@ use super::super::source::{ use super::super::version::{ ParsedVersion, compare_version_strings, is_stable_runtime_version, parse_version, }; +use super::uv_access::{ + UvAccess, UvPolicy, VersionResolution, plan_version_resolution, uv_for_compile, +}; use crate::runtime_registry::WheelRuntimeComposition; use anyhow::{Context, Result, bail}; use rocm_core::{ @@ -485,76 +488,6 @@ pub(in crate::therock) struct PipRuntimeResolution { pub(in crate::therock) published_device_targets: Vec, } -/// Whether a resolution may fetch `uv` to choose the ROCm 10 package versions. -/// -/// Only the ROCm 10 (`next`) layout needs `uv`: it has `uv pip compile` pick the -/// `torch` and `torchvision` builds. That is the one step of resolution that can -/// download the managed `uv` and create the data folder's `tools` folder. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(in crate::therock) enum UvAccess { - /// A real install or an update: take the managed `uv`, downloading it if no - /// copy is installed yet. This is what resolution has always done. - Acquire, - /// A preview: use a `uv` that is already there and never fetch one. - PresentOnly, -} - -/// What a resolution may do about `uv`: how far it may go ([`UvAccess`]) and the two -/// things it can ask for. The real ones, `find_uv_binary` and `ensure_uv_binary`, -/// are built where a resolution starts; they are fields so that a test can count -/// the calls instead of reaching the network. -struct UvPolicy<'a> { - access: UvAccess, - /// A `uv` that is already there, without acquiring one. - find_present: &'a dyn Fn() -> Option, - /// The managed `uv`, downloading it when none is installed. - acquire: &'a dyn Fn() -> Result, -} - -/// How a resolution chooses the package versions, given [`UvAccess`] and whether a -/// `uv` is present. A pure function of those inputs: see [`plan_version_resolution`]. -#[derive(Debug, Clone, PartialEq, Eq)] -enum VersionResolution { - /// The canonical layout: read each package's simple index. No `uv`. - ScrapeIndexes, - /// The ROCm 10 layout: run `uv pip compile` with this `uv`. - Compile(UvChoice), - /// The ROCm 10 layout with no `uv` to run and none to fetch: stop with what is - /// known without one, and leave the versions `uv` would choose unresolved. - Skip, -} - -/// Which `uv` the compile step runs. -#[derive(Debug, Clone, PartialEq, Eq)] -enum UvChoice { - /// Ask `ensure_uv_binary`, which downloads the managed `uv` when none is installed. - Acquire, - /// A `uv` that was already there. - Existing(PathBuf), -} - -/// Decide how the versions are chosen, without doing it. -/// -/// `find_present` looks for a `uv` that is already there. It runs only when the -/// answer depends on it: never for the canonical layout, which needs no `uv`, and -/// never for [`UvAccess::Acquire`], whose `ensure_uv_binary` does its own looking. -/// So a real install and an update decide, and call, exactly what they did before a -/// preview could skip the step, and no answer for them is [`VersionResolution::Skip`]. -fn plan_version_resolution( - layout: SourceLayout, - access: UvAccess, - find_present: impl FnOnce() -> Option, -) -> VersionResolution { - match (layout, access) { - (SourceLayout::Canonical, _) => VersionResolution::ScrapeIndexes, - (SourceLayout::Next, UvAccess::Acquire) => VersionResolution::Compile(UvChoice::Acquire), - (SourceLayout::Next, UvAccess::PresentOnly) => find_present() - .map_or(VersionResolution::Skip, |uv| { - VersionResolution::Compile(UvChoice::Existing(uv)) - }), - } -} - /// What a resolution knows once it has been asked for with a given [`UvAccess`]. #[derive(Debug, Clone)] pub(in crate::therock) enum PipRuntimePlan { @@ -1045,17 +978,6 @@ fn resolve_pip_runtime_from_index( })) } -/// The `uv` the compile step runs: the one already there, or the managed `uv` that -/// `acquire` (`ensure_uv_binary`) fetches when none is installed. -fn uv_for_compile(choice: UvChoice, acquire: impl FnOnce() -> Result) -> Result { - match choice { - UvChoice::Existing(uv) => Ok(uv), - UvChoice::Acquire => { - acquire().context("failed to acquire uv for ROCm X metadata resolution") - } - } -} - pub(in crate::therock) fn select_rocm_version( channel: TheRockChannel, rocm_versions: &[String], @@ -2544,115 +2466,7 @@ mod tests { } } - // ---- who may fetch `uv` ------------------------------------------------- - - /// Asks [`plan_version_resolution`] for one cell of the matrix and reports - /// the answer together with how many times a present `uv` was looked for. - fn decide( - layout: SourceLayout, - access: UvAccess, - uv_present: bool, - ) -> (VersionResolution, usize) { - let looked = std::cell::Cell::new(0); - let decision = plan_version_resolution(layout, access, || { - looked.set(looked.get() + 1); - uv_present.then(|| PathBuf::from("/opt/uv")) - }); - (decision, looked.get()) - } - - /// The state matrix of the decision: both layouts, both accesses, a `uv` - /// present or not. Only the ROCm 10 layout needs `uv`; a real install and an - /// update (`Acquire`) always go on to acquire it and never ask whether one is - /// present, because `ensure_uv_binary` does its own looking; a preview - /// (`PresentOnly`) asks, and with none present stops. `Skip` therefore exists - /// for exactly one row. - #[test] - fn the_version_step_is_decided_by_layout_access_and_a_present_uv_alone() { - use SourceLayout::{Canonical, Next}; - use UvAccess::{Acquire, PresentOnly}; - let existing = || VersionResolution::Compile(UvChoice::Existing(PathBuf::from("/opt/uv"))); - let rows = [ - // (layout, access, uv present, decision, times a present uv was looked for) - ( - Canonical, - Acquire, - false, - VersionResolution::ScrapeIndexes, - 0, - ), - ( - Canonical, - Acquire, - true, - VersionResolution::ScrapeIndexes, - 0, - ), - ( - Canonical, - PresentOnly, - false, - VersionResolution::ScrapeIndexes, - 0, - ), - ( - Canonical, - PresentOnly, - true, - VersionResolution::ScrapeIndexes, - 0, - ), - ( - Next, - Acquire, - false, - VersionResolution::Compile(UvChoice::Acquire), - 0, - ), - ( - Next, - Acquire, - true, - VersionResolution::Compile(UvChoice::Acquire), - 0, - ), - (Next, PresentOnly, false, VersionResolution::Skip, 1), - (Next, PresentOnly, true, existing(), 1), - ]; - for (layout, access, uv_present, expected, looked) in rows { - let (decision, calls) = decide(layout, access, uv_present); - let row = format!("{layout:?}, {access:?}, uv present: {uv_present}"); - assert_eq!(decision, expected, "decision for {row}"); - assert_eq!(calls, looked, "times a present uv was looked for in {row}"); - } - } - - /// The compile step runs the `uv` it was given, and asks for one only on - /// `Acquire`, with the context a failed fetch has always carried. - #[test] - fn the_compile_step_acquires_uv_only_when_told_to() { - let acquired = std::cell::Cell::new(0); - let acquire = || { - acquired.set(acquired.get() + 1); - Ok(PathBuf::from("/managed/uv")) - }; - assert_eq!( - uv_for_compile(UvChoice::Acquire, acquire).unwrap(), - PathBuf::from("/managed/uv") - ); - assert_eq!(acquired.get(), 1); - assert_eq!( - uv_for_compile(UvChoice::Existing(PathBuf::from("/opt/uv")), acquire).unwrap(), - PathBuf::from("/opt/uv") - ); - assert_eq!(acquired.get(), 1, "a present uv is not acquired again"); - - let error = uv_for_compile(UvChoice::Acquire, || bail!("no network")).unwrap_err(); - assert_eq!( - format!("{error:#}"), - "failed to acquire uv for ROCm X metadata resolution: no network" - ); - } + // ---- who may fetch `uv`: the real resolution --------------------------- /// Serves fixed pages over HTTP until dropped. A request for a path it does /// not hold is a 404. diff --git a/docs/architecture.md b/docs/architecture.md index d05957160b..cff7eb6724 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,7 +111,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/probe.rs`](../apps/rocm/src/therock/probe.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/process.rs` and `therock/python.rs` (split out of `therock.rs`): asking an installed runtime about itself through its own interpreter. `probe_rocm_sdk_runtime` runs `ROCM_SDK_PROBE_SCRIPT` and parses what `rocm_sdk` reports into the `RocmSdkPythonProbe` that [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) owns and validates; `probe_runtime_devices` runs `RUNTIME_DEVICE_PROBE_SCRIPT` to count the devices the runtime's torch can open and whether one can run a kernel (`RuntimeDeviceProbe`); `probe_torch_alignment` reads the installed torch and the one an engine pins from distribution metadata without importing torch (`TorchAlignmentProbe`); and `split_local_version`, `requirement_pinned_version` and `install_pinned_package` are the torch-pin helpers [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) uses to put the SDK's torch back after an engine install. The `pub(crate)` items are re-exported by `therock.rs` for [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs); `probe_rocm_sdk_runtime_for_target`, which `install_wheel_runtime` calls with the device payload it just chose, is `pub(super)`. - [`therock/resolve.rs`](../apps/rocm/src/therock/resolve.rs) — a child of `therock.rs` that holds no logic (split out of `therock.rs`): only the `mod` declarations for the artifact-resolution modules under `therock/resolve/` and the `pub(super) use` lines through which `therock.rs` reads them. It is the parent the `resolve/` files hang from, so a module there is `pub(in crate::therock)` for the items `therock.rs` reaches and private otherwise. - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. -- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. That compile is the one step that may fetch `uv`, so who may is decided in one pure function: `plan_version_resolution` takes the layout, a `UvAccess` and whether a `uv` is present, and answers scrape the indexes (canonical), compile with a `UvChoice` (acquire the managed `uv`, or use the present one) or skip. `resolve_pip_runtime` and `resolve_pip_runtime_with_timeout` (a real install and `rocm update`) are always `UvAccess::Acquire`, which never skips and never asks whether a `uv` is present; `preview_pip_runtime` is `UvAccess::PresentOnly`, which uses a present `uv` (`rocm_core::find_uv_binary`) and otherwise returns `PipRuntimePlan::NeedsUv` holding an `UnresolvedPipRuntime`: what was chosen without `uv` (the `rocm` version, the torchaudio pin, the requirements `uv` would be given) and no package versions, so nothing made from them can be built from it. The real `find_uv_binary` and `ensure_uv_binary` reach the resolution through a `UvPolicy` so that a test can count the calls. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. +- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. That compile is the one step that may fetch `uv`, so who may is decided in one pure function, [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs)'s `plan_version_resolution`, which takes the layout, a `UvAccess` and whether a `uv` is present, and answers scrape the indexes (canonical), compile with a `UvChoice` (acquire the managed `uv`, or use the present one) or skip. `resolve_pip_runtime` and `resolve_pip_runtime_with_timeout` (a real install and `rocm update`) are always `UvAccess::Acquire`, which never skips and never asks whether a `uv` is present; `preview_pip_runtime` is `UvAccess::PresentOnly`, which uses a present `uv` (`rocm_core::find_uv_binary`) and otherwise returns `PipRuntimePlan::NeedsUv` holding an `UnresolvedPipRuntime`: what was chosen without `uv` (the `rocm` version, the torchaudio pin, the requirements `uv` would be given) and no package versions, so nothing made from them can be built from it. `wheel.rs` acts on the answer; the real `find_uv_binary` and `ensure_uv_binary` reach it through a `UvPolicy` so that a test can count the calls. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. +- [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs) — full domain extraction, a leaf child of `therock/resolve.rs` that builds only on `therock/source.rs` for `SourceLayout` (split out of `therock/resolve/wheel.rs` to keep it within the module-size limit): who may fetch `uv` while a wheel stack is resolved. `UvAccess` says how far a resolution may go (`Acquire`, what a real install and `rocm update` pass, or `PresentOnly`, a preview's), `plan_version_resolution` turns that, the layout and whether a `uv` is present into a `VersionResolution` (scrape the indexes, compile with a `UvChoice`, or skip) without touching anything, and `uv_for_compile` hands the compile step the `uv` it runs, acquiring it only on `Acquire`, with the context a failed fetch has always carried. `UvPolicy` carries the access and the two real lookups (`find_uv_binary`, `ensure_uv_binary`) as closures so a test can count what a resolution asks for. Everything is `pub(super)` and read by `wheel.rs`; the state matrix over layout, access and presence lives in this module's tests. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. - [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) — full domain extraction, a leaf child of `therock.rs` (split out of `install_wheel_runtime`) that builds on `therock/python.rs` and `therock/resolve.rs` for the Python decision and the resolution it prints: the `sdk install` plan a wheel install prints, key by key. `WheelPlan` carries the facts, `write_wheel_plan` writes the lines a preview and a real install share, and `write_dry_run_tail` the lines only a preview adds (the `uv` commands it would run and the manifest path). `PlanStack` is the one place the plan differs: `Resolved` carries the package specs, runtime key, install folder and manifest path, and `NeedsUv` (a ROCm 10 preview on a host with no `uv`, see `UnresolvedPipRuntime` in `therock/resolve/wheel.rs`) prints those as `not resolved by this preview` and adds the `resolution_note:` line, so one writer serves both and a line the plan gains appears in both. `unresolved_wheel_preview` renders the second kind whole. `therock.rs` reads it through `pub(super)` items; it reads the provenance, host-note and `venv_python_path` helpers back from `therock.rs`. - [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. From 1c5a61c8c78c2ab4733e33fdb70fbe20c264d1b2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:52:17 +0000 Subject: [PATCH 06/11] docs: say that an install preview fetches no uv on any layout The notes written when the preview stopped preparing Python limited the no-uv, no-tools-folder guarantee to the canonical layout, because a ROCm 10 preview could still download the managed uv. It no longer does, so make every place that called the preview read-only or described its uv behaviour say what is now true: the --dry-run help, the README's ROCm 10 section, docs/testing.md and docs/manual-testing.md (including the new scenarios and the one gap no test reaches without a network), and the dash-TUI tool-group comment for install_sdk_dry_run. The assistant tool descriptions, docs/llm-tool-use.md and the assistant skill make no claim about the preview's uv handling, so they are unchanged. Signed-off-by: Jussi Elo --- README.md | 8 ++++ apps/rocm/src/cli.rs | 5 +++ crates/rocm-dash-tui/src/agent/tools.rs | 9 +++-- docs/manual-testing.md | 14 ++++--- docs/testing.md | 49 ++++++++++++++++++++++--- 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9473fba23b..c65567a5c6 100644 --- a/README.md +++ b/README.md @@ -543,6 +543,14 @@ a family of GPUs, or none), or you want to target a different GPU, also pass rocm install sdk --version 10.1.0 --family gfx1200 --dry-run ``` +The `--dry-run` preview downloads no `uv` and creates no `tools` folder. For ROCm +10 and newer, `uv pip compile` is what chooses the `torch` and `torchvision` +versions, so the preview runs a `uv` that is already installed and gives the +complete plan. With none installed it can't, and it says so in a +`resolution_note:` line: the package specs, runtime key, install folder, manifest +and `command:` (everything made from those versions) read `not resolved by this +preview`, and a real install fetches `uv` and resolves them. + A family label such as `--family gfx120X-all` is rejected for those versions instead of being resolved to a guess, because the ROCm 10 packages publish one payload per exact arch and there is no bucket payload to fall back to. Run diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 222a5ef726..f4855967d7 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -759,6 +759,11 @@ rocm install sdk --devel")] #[arg(long)] devel: bool, /// Resolve the install plan without changing files. + /// + /// Downloads no uv and prepares no Python. For ROCm 10 and newer on a + /// host with no uv installed, the plan marks the package versions it + /// would need uv to choose, and everything made from them, as not + /// resolved. #[arg(long)] dry_run: bool, /// Approve replacing the current active default ROCm runtime (and diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 0c96d22d6a..0cf7d08d56 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -211,9 +211,12 @@ pub const TELEMETRY_TOOL_NAMES: [&str; 4] = [ // `update_check` can also, on a host with no usable Python on PATH, download a // managed `uv` and Python into the data folder and record that Python in the // saved config; `install_sdk_dry_run` prepares no Python and saves no Python -// path: on such a host it reports the Python a real install would prepare. (A -// ROCm 10 preview still resolves package versions with `uv` and can fetch the -// managed `uv` into the data folder.) The boundary type +// path: on such a host it reports the Python a real install would prepare. It +// fetches no `uv` either: a ROCm 10 preview runs a `uv` that is already +// installed to resolve the package versions and, with none, prints the lines +// that depend on them as not resolved, so it creates no `tools` folder in the +// data folder. (`update_check` still resolves with `uv` and can fetch it.) The +// boundary type // (`SharedRocmToolExecutor`) is plain data — importing it does NOT pull // `rocm-core` into this crate; the `agent` module stays the sole `rig` namer. // --------------------------------------------------------------------------- diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 2171fdd3a5..28fc2a8cca 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -169,11 +169,15 @@ Expected result: plan's `command:` line shows `uv venv --python 3.12 ...` (a version, as no interpreter path exists yet), so it is not copy-pasteable verbatim: `uv` resolves `3.12` itself and may pick a different Python 3.12 than the managed - one a real install prepares. This holds for the canonical layout. A preview - pinned to ROCm 10 or newer (`--version 10.0.0 --family gfx1200`) still needs - `uv` to resolve the package versions, so on a clean host it can download the - managed `uv` and create the `tools` folder, although it prepares no Python and - saves no Python path. + one a real install prepares. This holds for every layout. A preview pinned + to ROCm 10 or newer (`--version 10.0.0 --family gfx1200`) needs `uv` to choose + the `torch` and `torchvision` versions, so on a host with no `uv` installed it + stops before that step: the data folder still has no `tools` folder, a + `resolution_note:` line says why, and `target:`, `runtime_key:`, + `package_specs:`, `command:` and `manifest:` read `not resolved by this + preview`. Run the real install (or any command that fetches `uv`), then the + preview again, and it prints the complete plan; so does a preview with + `ROCM_CLI_UV_BINARY` naming a `uv`. - With `ROCM_CLI_DISABLE_MANAGED_PYTHON_BOOTSTRAP=1`, or a `ROCM_CLI_MANAGED_PYTHON_VERSION` whose Python the layout does not accept (such as `3.13` for the canonical layout), on the same host the preview fails diff --git a/docs/testing.md b/docs/testing.md index 1cbf599274..71e732bef8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -763,12 +763,47 @@ download could start. The printed `command:` line then shows `uv venv --python 3.12 ...` (a version, because no interpreter path exists yet), so it is not copy-pasteable verbatim: `uv` resolves `3.12` itself, and on a host with a different Python 3.12 it can choose that one rather than the managed one -a real install prepares. That scenario is a canonical-layout preview. A preview -of a ROCm 10 or newer wheel (a `--version` pin of 10 or later) does still -need `uv`: it resolves the published package versions with `uv pip compile`, so -on a clean host it can download the managed `uv` and create the data folder's -`tools` folder, although it still prepares no Python. The "no `uv`, no `tools` -folder" guarantee is therefore the canonical layout's only. +a real install prepares. That scenario is a canonical-layout preview. + +A preview of a ROCm 10 or newer wheel (a `--version` pin of 10 or later) is held +to the same rule, and never fetches `uv` either. On that layout `uv pip compile` +is the one step that chooses the `torch` and `torchvision` versions (the CLI +picks only the ROCm version and the torchaudio pin), so the preview runs a `uv` +that is already there (the one `ROCM_CLI_UV_BINARY` names, or the managed copy a +previous install left) and, with none, stops before that step. It then prints +what needs no `uv` (the ROCm version, the torchaudio pin, the requirements `uv` +would be asked about), marks `target`, `runtime_key`, `package_specs`, `command` +and `manifest` as `not resolved by this preview`, and says why in a +`resolution_note:` line and a progress line, in the voice of `python_note:`. It +cannot say whether the published stack installs together either. A real install +and `rocm update` still fetch `uv` as before, and a preview made after either +has run is the complete plan. Scenario `runtime-28` checks the note against the +state it left on a host with no `uv`: no managed-tools folder, no tool archive in +its own cache folder (here the real detector, since no stand-in `uv` is named), +no saved config, and no `torch` or `torchvision` version anywhere in the output. +`runtime-27` is the other polarity: with a `uv` present (the recording stand-in, +which answers `uv pip compile` with the fixture's four pins, drains its stdin +under `FAKE_UV_DRAIN_STDIN`, and logs its arguments under `FAKE_UV_LOG`), the +preview runs it exactly once, for the compile, and prints the resolved plan. The +`therock-next` scenarios that print resolved package specs start from `And a uv +is already installed`, which installs the managed `uv` into the scenario's data +folder the way a real install would have. +The decision is unit-tested on every platform, Windows included +(`the_version_step_is_decided_by_layout_access_and_a_present_uv_alone` in +[uv_access.rs](../apps/rocm/src/therock/resolve/uv_access.rs): the state matrix of +both layouts, a preview or an install, and a `uv` present or not), as are the +plan's whole text and its key order against a resolved plan's +(`the_plan_a_preview_without_uv_prints_is_exactly_this` and +`a_resolved_and_an_unresolved_plan_print_the_same_keys` in +[wheel.rs](../apps/rocm/src/therock/resolve/wheel.rs)). On Unix, +`a_preview_never_acquires_uv_and_a_real_resolution_still_does` there runs the +real resolution against a served index and a stand-in `uv`, counting what each +row looked for, acquired and ran. What no test reaches without the network is the +real download a real install makes when no `uv` is installed, and that the public +entry points a real install and `rocm update` call pass `UvAccess::Acquire`: the +first is a network call, and the second is held only by the types (those entry +points return a resolved plan and refuse the unresolved one) and by the lanes +that install a ROCm 10 runtime for real. The preview does not announce a Python the install would refuse either, and it prints nothing that predicts what a real install would do on a path that ends in @@ -803,6 +838,8 @@ progress output and that `uv` was not run and no file was written. Beside it are `a_dry_run_refuses_a_managed_python_version_the_install_would_refuse` and `a_dry_run_reads_every_spelling_of_a_managed_python_version_the_same_way`. The preview does still write its package-index cache under the cache folder. +`update_check` and `rocm update --check` are not previews in this sense: they +resolve with `uv` and can fetch it, and prepare a managed Python too. Then it verifies: From 59582ea788f2832172328f3a19ab65c2ec0a70c3 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 21:00:16 +0000 Subject: [PATCH 07/11] test(install): satisfy clippy in the uv access tests and the plan helper Clippy under -D warnings flags a function that can be const, a test helper that appends a format! to a String, and an import that only the tests use since the decision moved to uv_access.rs. Fix the three; no behaviour changes. Signed-off-by: Jussi Elo --- apps/rocm/src/therock/resolve/wheel.rs | 7 +++++-- apps/rocm/src/therock/wheel_plan.rs | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index b8d6cb33d0..d06753ac61 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -59,7 +59,7 @@ use rocm_core::{ normalize_therock_family, runtime_is_windows, uv_command_env, }; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::process::{Command, Stdio}; use std::time::Duration; @@ -1512,6 +1512,7 @@ mod tests { use super::super::super::test_support::test_runtime_manifest; use super::*; use crate::runtime_registry::{InstalledRuntimeManifest, wheel_composition_includes_devel}; + use std::path::PathBuf; /// A representative slice of what the canonical aggregate index publishes. /// Notably it has no `gfx943`: MI300 steppings other than `gfx942` have no @@ -2539,9 +2540,11 @@ mod tests { #[cfg(unix)] fn wheel_links(package: &str, versions: &[&str]) -> String { + use std::fmt::Write as _; + versions.iter().fold(String::new(), |mut html, version| { let file = format!("{package}-{version}-py3-none-any.whl"); - html.push_str(&format!("{file}\n")); + let _ = writeln!(html, "{file}"); html }) } diff --git a/apps/rocm/src/therock/wheel_plan.rs b/apps/rocm/src/therock/wheel_plan.rs index 4deb719cd2..97fcb77e9b 100644 --- a/apps/rocm/src/therock/wheel_plan.rs +++ b/apps/rocm/src/therock/wheel_plan.rs @@ -78,7 +78,7 @@ pub(super) struct ResolvedStack<'a> { /// The `compatibility_key` the install is recorded under is the `rocm` build every /// package in the stack shares, so a plan that has not chosen the stack still /// knows it: the `rocm` version. -fn compatibility_key<'a>(plan: &'a WheelPlan<'a>) -> &'a str { +const fn compatibility_key<'a>(plan: &'a WheelPlan<'a>) -> &'a str { match &plan.stack { PlanStack::Resolved(stack) => stack.compatibility_key, PlanStack::NeedsUv { .. } => plan.latest_version, From 30848c909b38b0018b2fb5a45874433e72fdb69b Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 21:22:28 +0000 Subject: [PATCH 08/11] docs: name the real update check where the testing notes advise one docs/testing.md said rocm update --check does not preview in the install-preview sense, but rocm update has no --check flag: the plain rocm update (without --apply) is the check. The advised-commands test rejects the spelling. Name the update_check tool and the plain rocm update check instead. Signed-off-by: Jussi Elo --- docs/testing.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 71e732bef8..0b3165c0ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -838,8 +838,9 @@ progress output and that `uv` was not run and no file was written. Beside it are `a_dry_run_refuses_a_managed_python_version_the_install_would_refuse` and `a_dry_run_reads_every_spelling_of_a_managed_python_version_the_same_way`. The preview does still write its package-index cache under the cache folder. -`update_check` and `rocm update --check` are not previews in this sense: they -resolve with `uv` and can fetch it, and prepare a managed Python too. +The `update_check` tool and the plain `rocm update` check (no `--apply`) are not +previews in this sense: they resolve with `uv` and can fetch it, and prepare a +managed Python too. Then it verifies: From 3e0d2c57eff245b39cd2f7e8ae58b44e3a56e53b Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:42:05 +0000 Subject: [PATCH 09/11] docs(e2e): point runtime-28's comment at the test that holds the decision matrix The comment said the decision matrix and the plan wording are unit-tested in wheel.rs and wheel_plan.rs. The matrix test, the_version_step_is_decided_by_layout_access_and_a_present_uv_alone, lives in therock/resolve/uv_access.rs; the plan wording tests are in wheel.rs and exercise what wheel_plan.rs writes. docs/testing.md and docs/architecture.md already name the right files. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/features/runtime_setup.feature | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 6f365fd79c..f3032d6d66 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -621,8 +621,10 @@ Feature: Runtime configuration # # Linux only, for the same empty-`PATH` host as runtime-23. The decision is # unit-tested for every combination of layout, who may fetch `uv` and whether one - # is present in `apps/rocm/src/therock/resolve/wheel.rs`, and the plan's - # wording in `apps/rocm/src/therock/wheel_plan.rs`, on every platform. + # is present by `the_version_step_is_decided_by_layout_access_and_a_present_uv_alone` + # in `apps/rocm/src/therock/resolve/uv_access.rs`, and the plan's wording (written + # by `apps/rocm/src/therock/wheel_plan.rs`) by the tests in + # `apps/rocm/src/therock/resolve/wheel.rs`, on every platform. @id:runtime-install-preview-of-rocm-10-fetches-no-uv @requires-os:linux Scenario: runtime-28 - Previewing a ROCm 10 SDK install on a host with no uv fetches none and says what it did not resolve Given a canonical release pip index fixture and a ROCm 10 pip index fixture From 6b67113f0df6603b06bf20a74cdfe7042b3417d6 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:42:07 +0000 Subject: [PATCH 10/11] refactor(install): resolve a recorded device target in one place The rule that a recorded target (an update apply's recorded arch, or the one a runtime's manifest was built with) replaces the detected one, and is judged against the targets the source publishes, was written three times: in install_wheel_runtime, in the preview of a plan that stopped before uv, and in an update's plan. A later change to one would have missed the others. AggregateDeviceTarget::prefer_recorded is now the one place, and all three call it; behaviour is unchanged. Tests pin the rule itself (nothing recorded keeps the detected answer, undetermined included; a published target wins; an unpublished one is undetermined with its reason), show that the preview applies it through the helper, and scan the three files so that a call site that resolves a recorded target by hand again fails. Signed-off-by: Jussi Elo --- apps/rocm/src/therock.rs | 14 ++-- apps/rocm/src/therock/resolve/wheel.rs | 112 +++++++++++++++++++++++++ apps/rocm/src/therock/update.rs | 17 ++-- apps/rocm/src/therock/wheel_plan.rs | 14 ++-- docs/architecture.md | 2 +- 5 files changed, 129 insertions(+), 30 deletions(-) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 3b119d5de9..6418215338 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -436,15 +436,11 @@ fn install_wheel_runtime( include_devel, )? }; - let device_target = device_target_override.map_or_else( - || resolution.device_target.clone(), - |target| { - AggregateDeviceTarget::resolve( - Some(target), - &resolution.family, - &resolution.published_device_targets, - ) - }, + let device_target = AggregateDeviceTarget::prefer_recorded( + &resolution.device_target, + device_target_override, + &resolution.family, + &resolution.published_device_targets, ); // The exact device payload, not the version alone, decides what this runtime // can run, so it is what identifies the runtime. A preview on a host with no diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index d06753ac61..09371f73b8 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -200,6 +200,26 @@ impl AggregateDeviceTarget { Self::Exact(target) } + /// The device payload to plan for when the caller may already know the exact + /// target: an update apply's recorded arch, or the one a runtime's manifest was + /// built with. A recorded target wins over `detected` (the resolution's own + /// answer, from a probe of this host), and is checked against the targets the + /// source publishes exactly as a detected one is; with none recorded the + /// detected answer stands as it is, `Undetermined` included. The one place + /// that rule is written: a plan, a preview and an update's plan all call it, so + /// they cannot disagree about which target a runtime is for. + pub(in crate::therock) fn prefer_recorded( + detected: &Self, + recorded: Option<&str>, + family: &str, + published: &[String], + ) -> Self { + recorded.map_or_else( + || detected.clone(), + |target| Self::resolve(Some(target), family, published), + ) + } + pub(in crate::therock) fn as_str(&self) -> &str { match self { Self::Exact(target) => target, @@ -3060,4 +3080,96 @@ sdk install assert!(!resolved.contains("not resolved"), "{resolved}"); assert!(!resolved.contains("resolution_note"), "{resolved}"); } + + // ---- the recorded device target ------------------------------------------ + + /// The rule in one place: nothing recorded leaves the detected answer as it + /// was, a recorded target is judged against what the source publishes, and an + /// unpublished one is `Undetermined` with the reason rather than a guess. + #[test] + fn a_recorded_device_target_is_judged_against_what_the_source_publishes() { + let published = vec!["gfx1200".to_owned()]; + let undetected = AggregateDeviceTarget::Undetermined("no GPU".to_owned()); + let exact = AggregateDeviceTarget::Exact("gfx1200".to_owned()); + // Nothing recorded: the detected answer stands, undetermined included. + assert_eq!( + AggregateDeviceTarget::prefer_recorded(&undetected, None, "gfx120X-all", &published), + undetected + ); + assert_eq!( + AggregateDeviceTarget::prefer_recorded(&exact, None, "gfx120X-all", &published), + exact + ); + // Recorded and published: it wins over an undetermined host probe. + assert_eq!( + AggregateDeviceTarget::prefer_recorded( + &undetected, + Some("gfx1200"), + "gfx120X-all", + &published + ), + exact + ); + // Recorded but not published: undetermined, whatever the host probe said. + let refused = AggregateDeviceTarget::prefer_recorded( + &exact, + Some("gfx1201"), + "gfx120X-all", + &published, + ); + assert!( + matches!(&refused, AggregateDeviceTarget::Undetermined(reason) if reason.contains("gfx1201")), + "{refused:?}" + ); + } + + /// The same rule reaches a preview that stopped before `uv`: an override the + /// source does not publish shows as `` with its reason, not as the + /// target the resolution detected. Red if `unresolved_wheel_preview` stops + /// using the helper and keeps the detected target. + #[test] + fn a_preview_without_uv_applies_a_recorded_device_target_through_the_helper() { + let unresolved = unresolved_runtime(); + let (python, wheel_compatibility) = clean_host_python(); + let text = super::super::super::wheel_plan::unresolved_wheel_preview( + &super::super::super::wheel_plan::UnresolvedPreview { + channel: TheRockChannel::Release, + version_selector: None, + unresolved: &unresolved, + device_target_override: Some("gfx1201"), + prefix: None, + host_version_newer: None, + python: &python, + wheel_compatibility: &wheel_compatibility, + }, + ); + assert!( + text.contains("\n device_target: \n"), + "{text}" + ); + assert!(text.contains("\n device_target_reason: "), "{text}"); + assert!(!text.contains("device_target: gfx1200"), "{text}"); + } + + /// No call site writes the rule by hand again: every one goes through + /// `AggregateDeviceTarget::prefer_recorded`, so a change to the rule reaches + /// the install plan, the preview and an update's plan together. A scan of the + /// three files that call it, red if one resolves a recorded target itself. + #[test] + fn the_recorded_device_target_rule_is_written_once() { + for (name, source) in [ + ("therock.rs", include_str!("../../therock.rs")), + ("therock/update.rs", include_str!("../update.rs")), + ("therock/wheel_plan.rs", include_str!("../wheel_plan.rs")), + ] { + assert!( + source.contains("AggregateDeviceTarget::prefer_recorded("), + "{name} no longer resolves its device target through the helper" + ); + assert!( + !source.contains("AggregateDeviceTarget::resolve("), + "{name} resolves a device target by hand; use AggregateDeviceTarget::prefer_recorded" + ); + } + } } diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 5d49245313..c687d651c0 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -460,17 +460,12 @@ fn resolve_latest_for_manifest( // a fresh host probe. Planning must predict the key an apply will // produce, and re-probing would disagree with the installed runtime on // any host whose GPU is absent, hidden, or simply a second card. - let device_target = - wheel_composition_device_target(manifest.wheel_composition.as_ref()).map_or_else( - || resolution.device_target.clone(), - |target| { - AggregateDeviceTarget::resolve( - Some(target), - &resolution.family, - &resolution.published_device_targets, - ) - }, - ); + let device_target = AggregateDeviceTarget::prefer_recorded( + &resolution.device_target, + wheel_composition_device_target(manifest.wheel_composition.as_ref()), + &resolution.family, + &resolution.published_device_targets, + ); // No exact target means no reproducible composition, so freshness // falls back to the version comparison rather than demanding a repair // this host could not perform. diff --git a/apps/rocm/src/therock/wheel_plan.rs b/apps/rocm/src/therock/wheel_plan.rs index 97fcb77e9b..fe60fafad7 100644 --- a/apps/rocm/src/therock/wheel_plan.rs +++ b/apps/rocm/src/therock/wheel_plan.rs @@ -296,15 +296,11 @@ pub(super) fn unresolved_wheel_preview(preview: &UnresolvedPreview<'_>) -> Strin python, wheel_compatibility, } = *preview; - let device_target = device_target_override.map_or_else( - || unresolved.device_target.clone(), - |target| { - AggregateDeviceTarget::resolve( - Some(target), - &unresolved.family, - &unresolved.published_device_targets, - ) - }, + let device_target = AggregateDeviceTarget::prefer_recorded( + &unresolved.device_target, + device_target_override, + &unresolved.family, + &unresolved.published_device_targets, ); let explicit_install_root = prefix.map(rocm_core::resolve_path_through_symlinks); let no_wheel_warning = no_wheel_warning_for( diff --git a/docs/architecture.md b/docs/architecture.md index cff7eb6724..78c4dc3658 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,7 +111,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/probe.rs`](../apps/rocm/src/therock/probe.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/process.rs` and `therock/python.rs` (split out of `therock.rs`): asking an installed runtime about itself through its own interpreter. `probe_rocm_sdk_runtime` runs `ROCM_SDK_PROBE_SCRIPT` and parses what `rocm_sdk` reports into the `RocmSdkPythonProbe` that [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) owns and validates; `probe_runtime_devices` runs `RUNTIME_DEVICE_PROBE_SCRIPT` to count the devices the runtime's torch can open and whether one can run a kernel (`RuntimeDeviceProbe`); `probe_torch_alignment` reads the installed torch and the one an engine pins from distribution metadata without importing torch (`TorchAlignmentProbe`); and `split_local_version`, `requirement_pinned_version` and `install_pinned_package` are the torch-pin helpers [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) uses to put the SDK's torch back after an engine install. The `pub(crate)` items are re-exported by `therock.rs` for [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs); `probe_rocm_sdk_runtime_for_target`, which `install_wheel_runtime` calls with the device payload it just chose, is `pub(super)`. - [`therock/resolve.rs`](../apps/rocm/src/therock/resolve.rs) — a child of `therock.rs` that holds no logic (split out of `therock.rs`): only the `mod` declarations for the artifact-resolution modules under `therock/resolve/` and the `pub(super) use` lines through which `therock.rs` reads them. It is the parent the `resolve/` files hang from, so a module there is `pub(in crate::therock)` for the items `therock.rs` reaches and private otherwise. - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. -- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. That compile is the one step that may fetch `uv`, so who may is decided in one pure function, [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs)'s `plan_version_resolution`, which takes the layout, a `UvAccess` and whether a `uv` is present, and answers scrape the indexes (canonical), compile with a `UvChoice` (acquire the managed `uv`, or use the present one) or skip. `resolve_pip_runtime` and `resolve_pip_runtime_with_timeout` (a real install and `rocm update`) are always `UvAccess::Acquire`, which never skips and never asks whether a `uv` is present; `preview_pip_runtime` is `UvAccess::PresentOnly`, which uses a present `uv` (`rocm_core::find_uv_binary`) and otherwise returns `PipRuntimePlan::NeedsUv` holding an `UnresolvedPipRuntime`: what was chosen without `uv` (the `rocm` version, the torchaudio pin, the requirements `uv` would be given) and no package versions, so nothing made from them can be built from it. `wheel.rs` acts on the answer; the real `find_uv_binary` and `ensure_uv_binary` reach it through a `UvPolicy` so that a test can count the calls. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. +- [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. That compile is the one step that may fetch `uv`, so who may is decided in one pure function, [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs)'s `plan_version_resolution`, which takes the layout, a `UvAccess` and whether a `uv` is present, and answers scrape the indexes (canonical), compile with a `UvChoice` (acquire the managed `uv`, or use the present one) or skip. `resolve_pip_runtime` and `resolve_pip_runtime_with_timeout` (a real install and `rocm update`) are always `UvAccess::Acquire`, which never skips and never asks whether a `uv` is present; `preview_pip_runtime` is `UvAccess::PresentOnly`, which uses a present `uv` (`rocm_core::find_uv_binary`) and otherwise returns `PipRuntimePlan::NeedsUv` holding an `UnresolvedPipRuntime`: what was chosen without `uv` (the `rocm` version, the torchaudio pin, the requirements `uv` would be given) and no package versions, so nothing made from them can be built from it. `wheel.rs` acts on the answer; the real `find_uv_binary` and `ensure_uv_binary` reach it through a `UvPolicy` so that a test can count the calls. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`), and `AggregateDeviceTarget::prefer_recorded` is the one place a recorded target (an update apply's, or the one a runtime's manifest was built with) replaces the detected one, called by `therock.rs`, `therock/update.rs` and `therock/wheel_plan.rs`; `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/resolve/uv_access.rs`](../apps/rocm/src/therock/resolve/uv_access.rs) — full domain extraction, a leaf child of `therock/resolve.rs` that builds only on `therock/source.rs` for `SourceLayout` (split out of `therock/resolve/wheel.rs` to keep it within the module-size limit): who may fetch `uv` while a wheel stack is resolved. `UvAccess` says how far a resolution may go (`Acquire`, what a real install and `rocm update` pass, or `PresentOnly`, a preview's), `plan_version_resolution` turns that, the layout and whether a `uv` is present into a `VersionResolution` (scrape the indexes, compile with a `UvChoice`, or skip) without touching anything, and `uv_for_compile` hands the compile step the `uv` it runs, acquiring it only on `Acquire`, with the context a failed fetch has always carried. `UvPolicy` carries the access and the two real lookups (`find_uv_binary`, `ensure_uv_binary`) as closures so a test can count what a resolution asks for. Everything is `pub(super)` and read by `wheel.rs`; the state matrix over layout, access and presence lives in this module's tests. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. - [`therock/wheel_plan.rs`](../apps/rocm/src/therock/wheel_plan.rs) — full domain extraction, a leaf child of `therock.rs` (split out of `install_wheel_runtime`) that builds on `therock/python.rs` and `therock/resolve.rs` for the Python decision and the resolution it prints: the `sdk install` plan a wheel install prints, key by key. `WheelPlan` carries the facts, `write_wheel_plan` writes the lines a preview and a real install share, and `write_dry_run_tail` the lines only a preview adds (the `uv` commands it would run and the manifest path). `PlanStack` is the one place the plan differs: `Resolved` carries the package specs, runtime key, install folder and manifest path, and `NeedsUv` (a ROCm 10 preview on a host with no `uv`, see `UnresolvedPipRuntime` in `therock/resolve/wheel.rs`) prints those as `not resolved by this preview` and adds the `resolution_note:` line, so one writer serves both and a line the plan gains appears in both. `unresolved_wheel_preview` renders the second kind whole. `therock.rs` reads it through `pub(super)` items; it reads the provenance, host-note and `venv_python_path` helpers back from `therock.rs`. From 1ab6ce68e44142c783e5fd9a778d878787d19934 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:45:10 +0000 Subject: [PATCH 11/11] docs(install): tighten comments so wheel.rs stays within the module-size limit The recorded-device-target helper took therock/resolve/wheel.rs 13 production lines over the 1500-line limit. Shorten the doc comments on the helper, UnresolvedPipRuntime and the two resolver entry points; no code changes. Signed-off-by: Jussi Elo --- apps/rocm/src/therock/resolve/wheel.rs | 50 ++++++++++---------------- 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/apps/rocm/src/therock/resolve/wheel.rs b/apps/rocm/src/therock/resolve/wheel.rs index 09371f73b8..dc9c5ff937 100644 --- a/apps/rocm/src/therock/resolve/wheel.rs +++ b/apps/rocm/src/therock/resolve/wheel.rs @@ -16,10 +16,9 @@ //! `torchaudio_pin` pins a torchaudio first (the hardware-verified release when //! it is published) and `resolve_published_pip_package_versions` has //! `uv pip compile` choose the rest. That compile is the one step that may fetch -//! `uv`: a real install and an update (`resolve_pip_runtime`) acquire it, and a -//! preview (`preview_pip_runtime`) runs one that is already present and -//! otherwise stops with an `UnresolvedPipRuntime`, which is decided in -//! `plan_version_resolution` (in `uv_access`). `wheel_runtime_composition` turns a +//! `uv`: a real install and an update acquire it; a preview +//! (`preview_pip_runtime`) runs a present one or stops with an +//! `UnresolvedPipRuntime`, as `uv_access` decides. `wheel_runtime_composition` turns a //! resolution into the exact package specs (`therock_pip_package_specs`) an //! install gives `uv` and records in the manifest, and //! `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` @@ -200,14 +199,10 @@ impl AggregateDeviceTarget { Self::Exact(target) } - /// The device payload to plan for when the caller may already know the exact - /// target: an update apply's recorded arch, or the one a runtime's manifest was - /// built with. A recorded target wins over `detected` (the resolution's own - /// answer, from a probe of this host), and is checked against the targets the - /// source publishes exactly as a detected one is; with none recorded the - /// detected answer stands as it is, `Undetermined` included. The one place - /// that rule is written: a plan, a preview and an update's plan all call it, so - /// they cannot disagree about which target a runtime is for. + /// A recorded target (an update apply's, or a runtime manifest's) wins over + /// `detected` and is checked against what the source publishes; with none the + /// detected answer stands, `Undetermined` included. The one place this rule is + /// written, so a plan, a preview and an update's plan cannot disagree. pub(in crate::therock) fn prefer_recorded( detected: &Self, recorded: Option<&str>, @@ -517,17 +512,11 @@ pub(in crate::therock) enum PipRuntimePlan { NeedsUv(UnresolvedPipRuntime), } -/// A ROCm 10 resolution stopped before `uv pip compile`: everything chosen without -/// `uv`, and none of what depends on it. -/// -/// Known: the family, the device target, the index, the `rocm` version and the -/// torchaudio pin, and the requirement lines `uv` would be asked to resolve. Not -/// known: the `torch` and `torchvision` versions, which only `uv pip compile` -/// produces, nor whether the published stack is jointly installable. Everything -/// built from them (the package specs, the runtime key, the install folder and -/// manifest the key names) is therefore unknown too, which is why this carries no -/// [`TheRockPipPackageVersions`]: a value that cannot be made cannot be mistaken -/// for one. +/// A ROCm 10 resolution stopped before `uv pip compile`: what was chosen without +/// `uv` (family, device target, index, `rocm` version, torchaudio pin, the +/// requirements `uv` would be given) and nothing made from the `torch` and +/// `torchvision` versions only `uv` produces. It carries no +/// [`TheRockPipPackageVersions`], so such a value cannot be mistaken for one. #[derive(Debug, Clone)] pub(in crate::therock) struct UnresolvedPipRuntime { pub(in crate::therock) family: String, @@ -684,9 +673,8 @@ pub(in crate::therock) fn resolve_pip_runtime( /// `layout_override` is how an update keeps a runtime on the stream it was /// installed from; a fresh install passes `None` and lets the request decide. /// -/// Always [`UvAccess::Acquire`]: a real install and an update take the managed `uv` -/// when the ROCm 10 layout needs one, as they always have. A preview asks -/// [`preview_pip_runtime`] instead. +/// Always [`UvAccess::Acquire`], as a real install and an update always were; a +/// preview asks [`preview_pip_runtime`]. #[allow(clippy::too_many_arguments)] pub(in crate::therock) fn resolve_pip_runtime_with_timeout( paths: &AppPaths, @@ -710,18 +698,16 @@ pub(in crate::therock) fn resolve_pip_runtime_with_timeout( UvAccess::Acquire, )? { PipRuntimePlan::Resolved(resolution) => Ok(resolution), - // `plan_version_resolution` answers `Skip` only to `UvAccess::PresentOnly`, - // and a test holds it to that, so this is a broken invariant, not a state. + // `Skip` is only ever answered to `PresentOnly` (a test holds it to that). PipRuntimePlan::NeedsUv(_) => { bail!("internal error: a resolution that may acquire uv stopped for want of one") } } } -/// [`resolve_pip_runtime`] for a preview: it never downloads `uv` or creates the -/// folders that hold it. A `uv` that is already there is used, so the answer is the -/// resolution a real install would reach; with none, a ROCm 10 request comes back as -/// [`PipRuntimePlan::NeedsUv`], holding what `uv` is not needed for. +/// [`resolve_pip_runtime`] for a preview: never downloads `uv` or creates the +/// folders that hold it. A `uv` that is already there is used; with none, a ROCm 10 +/// request comes back as [`PipRuntimePlan::NeedsUv`]. pub(in crate::therock) fn preview_pip_runtime( paths: &AppPaths, channel: TheRockChannel,