From 16587cc3db0583f37a4a62ff9602b719d1a00f4c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 16:12:37 +0000 Subject: [PATCH 01/20] test: pin how each caller finds a Python environment root from its interpreter Three private helpers turn an interpreter path into the environment folder that contains it, and they do not agree. ComfyUI's runtime environment returns a root only when the interpreter's folder is bin or Scripts (any case) and otherwise leaves VIRTUAL_ENV unset. The adopt command and the vLLM launch both take the grandparent whatever the folder is called: the bin/Scripts check in the adopt helper returns the same value on both of its branches, and vLLM never had one. So an interpreter in an unrelated folder is guessed at by two callers and refused by the third. Add tests that pass on the unmodified code and pin each caller's behaviour for bin, Scripts, SCRIPTS, an unrelated folder, a bare file name and a root-level interpreter, including those differences, so the change that makes the callers agree shows exactly what moves. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 60 ++++++++++++++++++++++++++++ apps/rocm/src/runtime_adopt.rs | 56 ++++++++++++++++++++++++++ engines/vllm/src/process.rs | 26 ++++++++++++ 3 files changed, 142 insertions(+) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 3e496900be..ba0f247f20 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -964,4 +964,64 @@ mod tests { .find(|(name, _)| name.to_string_lossy() == key) .and_then(|(_, value)| value.map(OsString::from)) } + + /// The environment folder ComfyUI exports as `VIRTUAL_ENV`, by the folder + /// the interpreter sits in. Today only a `bin` or `Scripts` folder + /// (any case) identifies a venv; anything else leaves `VIRTUAL_ENV` unset + /// rather than pointing it at a guessed folder. + #[test] + fn runtime_environment_exports_virtual_env_only_for_a_bin_or_scripts_folder() -> Result<()> { + let cases: [(&str, Option<&str>); 7] = [ + ("/opt/venv/bin/python", Some("/opt/venv")), + ("/opt/venv/Scripts/python.exe", Some("/opt/venv")), + ("/opt/venv/SCRIPTS/python.exe", Some("/opt/venv")), + ("/opt/venv/BIN/python", Some("/opt/venv")), + ("/opt/venv/tools/python", None), + ("python", None), + ("/python", None), + ]; + for (python, expected) in cases { + let python = PathBuf::from(python); + let manifest = minimal_runtime_manifest(&python); + + let env = runtime_environment_from_runtime(&manifest, &python); + let mut command = Command::new(&python); + apply_runtime_environment(&mut command, &env)?; + + assert_eq!( + command_env_value(&command, "VIRTUAL_ENV").as_deref(), + expected.map(std::ffi::OsStr::new), + "{}", + python.display() + ); + } + Ok(()) + } + + fn minimal_runtime_manifest(python: &Path) -> therock::InstalledRuntimeManifest { + therock::InstalledRuntimeManifest { + runtime_key: "therock-release-gfx120x-all".to_owned(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + channel: "release".to_owned(), + format: "wheel".to_owned(), + family: "gfx120X-all".to_owned(), + family_source: "test".to_owned(), + version: "7.13.0a20260511".to_owned(), + install_root: PathBuf::from("/nonexistent/runtime"), + selected_artifact_url: "https://example.invalid/simple".to_owned(), + source_layout_generation: None, + index_url: None, + tarball_file_name: None, + python_launcher: None, + python_executable: Some(python.display().to_string()), + pip_cache_dir: None, + rocm_sdk: None, + sdk_torch: None, + wheel_composition: None, + read_only: false, + imported_from: None, + devel: true, + installed_at_unix_ms: 100, + } + } } diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 549753818a..d6f47b1425 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -819,4 +819,60 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// Today's rule for `rocm runtimes adopt --python ` without `--root`: + /// the environment folder is the interpreter's grandparent no matter what + /// the interpreter's own folder is called. The `bin`/`Scripts` check in + /// `infer_python_env_root` returns the same value on both branches, so an + /// interpreter in an unrelated folder is guessed at rather than refused. + #[test] + fn adopt_python_file_infers_the_grandparent_whatever_the_folder_is_called() -> Result<()> { + let (root, _) = test_paths("runtime-adopt-infer-root"); + for folder in ["bin", "Scripts", "SCRIPTS", "tools"] { + let env_root = root.join(format!("env-{folder}")); + let python = env_root.join(folder).join("python"); + fs::create_dir_all(python.parent().unwrap())?; + fs::write(&python, "python")?; + + let (resolved, inferred) = resolve_adopt_python_input(&python)?; + + assert_eq!(resolved, python, "{folder}"); + assert_eq!(inferred.as_deref(), Some(env_root.as_path()), "{folder}"); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn adopt_infer_python_env_root_gives_up_only_without_a_grandparent() { + // A bare file name has an empty parent and a root-level file's parent + // is the filesystem root: neither names a folder to take the parent of. + assert_eq!(infer_python_env_root(Path::new("python")), None); + assert_eq!(infer_python_env_root(Path::new("/python")), None); + // One level deep the folder's own name is ignored. + assert_eq!( + infer_python_env_root(Path::new("/opt/venv/tools/python")).as_deref(), + Some(Path::new("/opt/venv")) + ); + } + + /// What the `None` above costs the user: the request is refused and asks + /// for `--root`, so nothing is registered. + #[test] + fn adopt_request_without_an_inferred_root_asks_for_root() { + let error = infer_adopt_runtime_request( + PathBuf::from("python"), + None, + None, + None, + None, + false, + &therock::RocmSdkPythonProbe::default(), + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("pass --root"), "{error}"); + } } diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index 98ccd5ead5..d8b6cdf933 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -1124,6 +1124,32 @@ mod tests { PathBuf::from("/opt/venv") ); } + /// Today's rule for a runtime's venv: the interpreter's grandparent + /// whatever its own folder is called, refused only when there is no + /// grandparent at all. + #[test] + fn vllm_venv_root_takes_the_grandparent_whatever_the_folder_is_called() { + for (python, expected) in [ + ("/opt/venv/bin/python", Some("/opt/venv")), + ("/opt/venv/Scripts/python.exe", Some("/opt/venv")), + ("/opt/venv/SCRIPTS/python.exe", Some("/opt/venv")), + ("/opt/venv/tools/python", Some("/opt/venv")), + ("python", None), + ("/python", None), + ] { + let runtime = test_vllm_runtime(Some(PathBuf::from(python)), None); + let root = vllm_venv_root(&runtime); + match expected { + Some(expected) => { + assert_eq!(root.unwrap(), PathBuf::from(expected), "{python}"); + } + None => { + let error = root.expect_err(python).to_string(); + assert!(error.contains("is not inside a venv"), "{python}: {error}"); + } + } + } + } #[test] fn venv_site_packages_dir_finds_the_single_python3_dir() -> Result<()> { let venv = tempfile::tempdir()?; From 7157d087dcca5b963b807d9083565b7e50b92a34 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 16:39:33 +0000 Subject: [PATCH 02/20] refactor: find a Python environment root from its interpreter by one rule The adopt command, the ComfyUI runtime environment and the vLLM launch each kept a private copy of "which environment folder holds this interpreter", and the copies disagreed. ComfyUI gave a root only for an interpreter in a bin or Scripts folder; adopt and vLLM took the grandparent whatever the folder was called (adopt's bin/Scripts check returned the same value on both branches). An interpreter in an unrelated folder was refused by one caller and guessed at by the other two. Add runtime_python_env_root_for_executable to rocm-core, beside the helpers it inverts, and use it from all three. An interpreter belongs to an environment only when it sits directly in a bin or Scripts folder, in any case; everything else is None, because a guessed folder is not an environment. The engine crate cannot reach apps/rocm, so rocm-core is the shared place. Valid venv layouts resolve exactly as before. Two things change for an interpreter outside that layout: `rocm runtimes adopt --python` without --root now stops and asks for --root instead of adopting the interpreter's grandparent, and the vLLM launch fails with its existing "not inside a venv" error instead of searching the wrong folder. Each caller's test goes through the shared function, scenario runtime-23 covers the adopt refusal end to end, and the --root help text, README and docs/testing.md now say how the folder is found. Signed-off-by: Jussi Elo --- README.md | 4 +- apps/rocm/src/comfyui/runtime_env.rs | 15 +-- apps/rocm/src/main.rs | 2 +- apps/rocm/src/runtime_adopt.rs | 94 ++++++++++--------- crates/rocm-core/src/lib.rs | 3 +- crates/rocm-core/src/runtime.rs | 76 +++++++++++++++ docs/architecture.md | 2 +- docs/testing.md | 4 + engines/vllm/src/process.rs | 41 ++++---- .../features/runtime_setup.feature | 19 ++++ tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 63 +++++++++++++ 11 files changed, 239 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 340d3c8a52..621a328a7b 100644 --- a/README.md +++ b/README.md @@ -712,7 +712,9 @@ exits without prompting or making changes. `adopt` registers an existing TheRock-based Python environment as a managed runtime. It does not work with standard ROCm package installs (for example, -`/opt/rocm`); use `rocm install sdk` instead. +`/opt/rocm`); use `rocm install sdk` instead. Without `--root`, the environment +folder is the one that holds the `--python` interpreter's `bin` (or `Scripts`) +folder; an interpreter anywhere else is refused until you pass `--root`. ### Disk space diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index ba0f247f20..babb58f48e 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -22,7 +22,7 @@ use crate::therock; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, RocmCliConfig, runtime_is_linux, runtime_is_windows, runtime_path_list_join, - runtime_path_list_split, runtime_paths_equivalent, + runtime_path_list_split, runtime_paths_equivalent, runtime_python_env_root_for_executable, }; use std::ffi::OsString; use std::path::{Path, PathBuf}; @@ -237,7 +237,7 @@ pub(super) fn runtime_environment_for_manifest( } let mut env = ComfyUiRuntimeEnvironment { - venv_root: infer_python_env_root(&manifest.python_executable), + venv_root: runtime_python_env_root_for_executable(&manifest.python_executable), rocm_root: Some(manifest.runtime_root.clone()), path_entries: Vec::new(), library_entries: Vec::new(), @@ -254,7 +254,7 @@ pub(super) fn runtime_environment_from_runtime( python: &Path, ) -> ComfyUiRuntimeEnvironment { let mut env = ComfyUiRuntimeEnvironment { - venv_root: infer_python_env_root(python), + venv_root: runtime_python_env_root_for_executable(python), rocm_root: manifest .rocm_sdk .as_ref() @@ -307,15 +307,6 @@ fn collect_rocm_runtime_paths(root: &Path, env: &mut ComfyUiRuntimeEnvironment) } } -fn infer_python_env_root(python: &Path) -> Option { - let bin_dir = python.parent()?; - let name = bin_dir.file_name()?.to_string_lossy().to_ascii_lowercase(); - if matches!(name.as_str(), "scripts" | "bin") { - return bin_dir.parent().map(Path::to_path_buf); - } - None -} - pub(super) fn apply_runtime_environment( command: &mut Command, env: &ComfyUiRuntimeEnvironment, diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index ca1d801976..55dd8be291 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1047,7 +1047,7 @@ left, so it cannot undo more than one activation." /// Python executable inside the existing ROCm environment. #[arg(long)] python: PathBuf, - /// Optional SDK root path. + /// Python environment folder. Taken from --python when that sits in a bin or Scripts folder; required otherwise. #[arg(long)] root: Option, /// Runtime id to assign. diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index d6f47b1425..d4a1e82388 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -19,7 +19,10 @@ use std::fs; use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_executable_in_env}; +use rocm_core::{ + AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_env_root_for_executable, + runtime_python_executable_in_env, +}; use crate::runtime_registry::{ paths_equivalent, runtime_manifest_path, runtime_registry_dir, @@ -230,7 +233,9 @@ fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option) return Ok((python, Some(env_root))); } if absolute.is_file() { - let inferred_root = infer_python_env_root(&absolute); + // `None` for an interpreter outside a `bin`/`Scripts` folder: the + // request then asks for `--root` rather than adopting a guessed folder. + let inferred_root = runtime_python_env_root_for_executable(&absolute); return Ok((absolute, inferred_root)); } bail!( @@ -239,15 +244,6 @@ fn resolve_adopt_python_input(input: &Path) -> Result<(PathBuf, Option) ); } -fn infer_python_env_root(python_executable: &Path) -> Option { - let bin_dir = python_executable.parent()?; - let bin_name = bin_dir.file_name()?.to_string_lossy(); - if bin_name.eq_ignore_ascii_case("Scripts") || bin_name == "bin" { - return bin_dir.parent().map(Path::to_path_buf); - } - bin_dir.parent().map(Path::to_path_buf) -} - pub(crate) fn adopt_runtime_from_probe( paths: &AppPaths, request: AdoptRuntimeRequest, @@ -820,15 +816,22 @@ mod tests { Ok(()) } - /// Today's rule for `rocm runtimes adopt --python ` without `--root`: - /// the environment folder is the interpreter's grandparent no matter what - /// the interpreter's own folder is called. The `bin`/`Scripts` check in - /// `infer_python_env_root` returns the same value on both branches, so an - /// interpreter in an unrelated folder is guessed at rather than refused. + /// `rocm runtimes adopt --python ` without `--root` takes the + /// environment folder from the shared rule (see + /// `rocm_core::runtime_python_env_root_for_executable`): only an + /// interpreter in a `bin` or `Scripts` folder, in any case, names one. An + /// interpreter in an unrelated folder yields no root, and the request is + /// then refused with `--root` guidance rather than adopting a guessed + /// folder. #[test] - fn adopt_python_file_infers_the_grandparent_whatever_the_folder_is_called() -> Result<()> { + fn adopt_python_file_infers_a_root_only_from_a_bin_or_scripts_folder() -> Result<()> { let (root, _) = test_paths("runtime-adopt-infer-root"); - for folder in ["bin", "Scripts", "SCRIPTS", "tools"] { + for (folder, expected_root) in [ + ("bin", true), + ("Scripts", true), + ("SCRIPTS", true), + ("tools", false), + ] { let env_root = root.join(format!("env-{folder}")); let python = env_root.join(folder).join("python"); fs::create_dir_all(python.parent().unwrap())?; @@ -837,42 +840,43 @@ mod tests { let (resolved, inferred) = resolve_adopt_python_input(&python)?; assert_eq!(resolved, python, "{folder}"); - assert_eq!(inferred.as_deref(), Some(env_root.as_path()), "{folder}"); + assert_eq!( + inferred.as_deref(), + expected_root.then_some(env_root.as_path()), + "{folder}" + ); } let _ = fs::remove_dir_all(root); Ok(()) } - #[test] - fn adopt_infer_python_env_root_gives_up_only_without_a_grandparent() { - // A bare file name has an empty parent and a root-level file's parent - // is the filesystem root: neither names a folder to take the parent of. - assert_eq!(infer_python_env_root(Path::new("python")), None); - assert_eq!(infer_python_env_root(Path::new("/python")), None); - // One level deep the folder's own name is ignored. - assert_eq!( - infer_python_env_root(Path::new("/opt/venv/tools/python")).as_deref(), - Some(Path::new("/opt/venv")) - ); - } - - /// What the `None` above costs the user: the request is refused and asks - /// for `--root`, so nothing is registered. + /// What no inferred root costs the user: the request is refused and asks + /// for `--root`, and an explicit `--root` still adopts the same + /// interpreter. #[test] fn adopt_request_without_an_inferred_root_asks_for_root() { - let error = infer_adopt_runtime_request( - PathBuf::from("python"), - None, - None, - None, - None, - false, - &therock::RocmSdkPythonProbe::default(), - ) - .unwrap_err() - .to_string(); + let python = PathBuf::from("python"); + let request = |install_root: Option| { + infer_adopt_runtime_request( + python.clone(), + install_root, + None, + None, + None, + false, + &therock::RocmSdkPythonProbe { + rocm_sdk_version: Some("7.13.0".to_owned()), + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }, + ) + }; + let error = request(None).unwrap_err().to_string(); assert!(error.contains("pass --root"), "{error}"); + + let explicit = request(Some(PathBuf::from("/opt/rocm-env"))).unwrap(); + assert_eq!(explicit.install_root, PathBuf::from("/opt/rocm-env")); } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 07d1c1eebf..f290fd4b1b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -205,7 +205,8 @@ pub use runtime::{ runtime_path_sort_key, runtime_path_text_is_absolute_for_host, runtime_path_text_is_absolute_for_platform, runtime_paths_equivalent, runtime_python_activation_hint, runtime_python_activation_script, runtime_python_bin_dir_name, - runtime_python_env_bin_dir, runtime_python_executable_in_env, runtime_python_executable_name, + runtime_python_env_bin_dir, runtime_python_env_root_for_executable, + runtime_python_executable_in_env, runtime_python_executable_name, runtime_rocm_library_filename, shell_command_for_host, user_runtime_dir, }; pub use service_identity::{ServiceId, generate_service_id}; diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index 3535035b07..b6a31d4f93 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -119,6 +119,31 @@ pub fn runtime_python_executable_in_env(env_root: &Path) -> PathBuf { runtime_python_env_bin_dir(env_root).join(runtime_python_executable_name()) } +/// The environment folder an interpreter belongs to, found from the +/// interpreter's own path: the inverse of [`runtime_python_executable_in_env`] +/// for a caller that holds only the interpreter. +/// +/// An interpreter belongs to an environment only when it sits directly in that +/// environment's `bin` or `Scripts` folder, and the answer is that folder's +/// parent. Both layouts are accepted, in any letter case, whatever the host: a +/// recorded interpreter path is not always from this host's layout, and a +/// Windows file system does not tell `Scripts` from `SCRIPTS`. +/// +/// Anything else gives `None`: an interpreter in some other folder, a bare file +/// name, or one at the file system root. A guess would name a folder that is +/// not an environment, and the caller would then export it as `VIRTUAL_ENV`, +/// look for `lib/python3.*` under it, or register it as a runtime. `None` lets +/// each caller refuse in its own words instead. +pub fn runtime_python_env_root_for_executable(python: &Path) -> Option { + let bin_dir = python.parent()?; + let name = bin_dir.file_name()?.to_str()?; + if name.eq_ignore_ascii_case("bin") || name.eq_ignore_ascii_case("Scripts") { + bin_dir.parent().map(Path::to_path_buf) + } else { + None + } +} + pub fn runtime_python_activation_script(env_root: &Path) -> PathBuf { let script = if runtime_is_windows() { "activate.bat" @@ -1026,6 +1051,57 @@ mod tests { use super::*; use std::fs; + /// The one rule every caller shares. Paths are built with `join`, not + /// written with a separator, so the table means the same on every host. + #[test] + fn env_root_for_executable_needs_a_bin_or_scripts_folder() { + let venv = Path::new("opt").join("venv"); + for (folder, python) in [ + ("bin", "python"), + ("BIN", "python"), + ("Scripts", "python.exe"), + ("SCRIPTS", "python.exe"), + ("scripts", "python.exe"), + ] { + assert_eq!( + runtime_python_env_root_for_executable(&venv.join(folder).join(python)), + Some(venv.clone()), + "{folder}" + ); + } + for folder in ["tools", "bin2", "my-bin", "venv"] { + assert_eq!( + runtime_python_env_root_for_executable(&venv.join(folder).join("python")), + None, + "{folder}" + ); + } + // A bare file name has an empty parent, and a root-level file's parent + // is the file system root: neither is inside a `bin` or `Scripts`. + assert_eq!( + runtime_python_env_root_for_executable(Path::new("python")), + None + ); + assert_eq!( + runtime_python_env_root_for_executable( + &Path::new(std::path::MAIN_SEPARATOR_STR).join("python") + ), + None + ); + } + + /// The inverse really is the inverse: the interpreter this host's layout + /// puts in an environment leads back to that environment. + #[test] + fn env_root_for_executable_inverts_the_hosts_layout() { + let env_root = Path::new("opt").join("venv"); + let python = runtime_python_executable_in_env(&env_root); + assert_eq!( + runtime_python_env_root_for_executable(&python).as_deref(), + Some(normalize_runtime_path_for_host(&env_root).as_path()) + ); + } + fn scratch_dir(name: &str) -> PathBuf { let root = std::env::temp_dir().join(format!( "rocm-core-resolve-{name}-{}-{:?}", diff --git a/docs/architecture.md b/docs/architecture.md index d37b0bb91c..e04dee750c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -216,7 +216,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. -- [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default config/data/cache directories, and path-text and path-list helpers (no module doc comment). +- [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, and path-text and path-list helpers (no module doc comment). - [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose.rs`). - [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. - [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. diff --git a/docs/testing.md b/docs/testing.md index 5497fac09f..9a69c0cca3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -907,6 +907,10 @@ rocm runtimes adopt \ --runtime-key adopted-release-pip-gfx120x-all-7-13-0 ``` +`--root` may be left out when `--python` sits directly in the environment's `bin` +(or `Scripts`) folder; an interpreter anywhere else is refused until `--root` is +given (scenario `runtime-23`). + Focused unit coverage: ```bash diff --git a/engines/vllm/src/process.rs b/engines/vllm/src/process.rs index d8b6cdf933..b382da9b98 100644 --- a/engines/vllm/src/process.rs +++ b/engines/vllm/src/process.rs @@ -459,16 +459,12 @@ fn vllm_venv_root(runtime: &VllmRuntime) -> Result { runtime.runtime_id ) })?; - python - .parent() - .and_then(Path::parent) - .map(Path::to_path_buf) - .ok_or_else(|| { - anyhow!( - "python executable `{}` is not inside a venv (bin/Scripts) layout", - python.display() - ) - }) + rocm_core::runtime_python_env_root_for_executable(python).ok_or_else(|| { + anyhow!( + "python executable `{}` is not inside a venv (bin/Scripts) layout", + python.display() + ) + }) } /// Finds `/lib/python3.*/site-packages`, the one location a venv keeps /// its packages under. Bails on anything other than exactly one `python3.*` @@ -1124,29 +1120,28 @@ mod tests { PathBuf::from("/opt/venv") ); } - /// Today's rule for a runtime's venv: the interpreter's grandparent - /// whatever its own folder is called, refused only when there is no - /// grandparent at all. + /// The runtime's venv is found by the rule every caller shares (see + /// `rocm_core::runtime_python_env_root_for_executable`): an interpreter in + /// an unrelated folder is refused, not mapped to that folder's parent, + /// which would send the site-packages lookup into a folder that is not a + /// venv. #[test] - fn vllm_venv_root_takes_the_grandparent_whatever_the_folder_is_called() { + fn vllm_venv_root_needs_a_bin_or_scripts_folder() { for (python, expected) in [ ("/opt/venv/bin/python", Some("/opt/venv")), ("/opt/venv/Scripts/python.exe", Some("/opt/venv")), ("/opt/venv/SCRIPTS/python.exe", Some("/opt/venv")), - ("/opt/venv/tools/python", Some("/opt/venv")), + ("/opt/venv/tools/python", None), ("python", None), ("/python", None), ] { let runtime = test_vllm_runtime(Some(PathBuf::from(python)), None); let root = vllm_venv_root(&runtime); - match expected { - Some(expected) => { - assert_eq!(root.unwrap(), PathBuf::from(expected), "{python}"); - } - None => { - let error = root.expect_err(python).to_string(); - assert!(error.contains("is not inside a venv"), "{python}: {error}"); - } + if let Some(expected) = expected { + assert_eq!(root.unwrap(), PathBuf::from(expected), "{python}"); + } else { + let error = root.expect_err(python).to_string(); + assert!(error.contains("is not inside a venv"), "{python}: {error}"); } } } diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 2879432000..e5f59f3fdf 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -473,3 +473,22 @@ Feature: Runtime configuration When the user approves a natural-language SDK install into a folder whose name has a quote Then the request plan names that folder as a single argument And the executed command names that folder as a single argument + + # `rocm runtimes adopt --python` takes the environment folder from where the + # interpreter sits: the folder that holds its `bin` (or `Scripts`) folder. An + # interpreter anywhere else has no environment folder to infer, and a guess (the + # interpreter's grandparent, which is what the command used to adopt) would + # register a folder that is not an environment as a runtime. The command refuses + # instead and names the flag that supplies the folder; nothing is saved. + # + # Linux-only: the stand-in interpreter is a POSIX shell script. The "adopts with + # --root" half is covered by the unit test + # `adopt_request_without_an_inferred_root_asks_for_root`, which shows an explicit + # root clears the refusal; adopting for real needs a genuine ROCm SDK environment. + @id:runtime-adopt-outside-a-venv-asks-for-root @requires-os:linux + Scenario: runtime-23 - Adopting a Python that sits outside any environment's bin folder asks for --root + Given a Python interpreter that sits outside any environment's bin folder + When the user adopts that interpreter without naming an environment folder + Then the adoption is refused + And the refusal asks for the environment folder with --root + And no runtime is registered diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 4150c9a679..6cf219fd41 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -1515,3 +1515,66 @@ async fn assert_freeform_execution_prefix_is_one_word(world: &mut E2eWorld) { &freeform_prefix_with_quote(world), ); } + +/// Where scenario runtime-23 plants its interpreter: inside a folder called `tools`, +/// which is neither `bin` nor `Scripts`, so nothing about its location names an +/// environment. +fn interpreter_outside_a_venv(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("not-a-venv") + .join("tools") + .join("python") +} + +/// A stand-in interpreter that answers the SDK probe with the smallest document the +/// CLI parses. The refusal under test happens after the probe and before anything is +/// read from the environment, so nothing else about the interpreter matters. +/// +/// A POSIX shell script, so the scenario is Linux-only (`@requires-os:linux`). +#[given("a Python interpreter that sits outside any environment's bin folder")] +async fn setup_interpreter_outside_a_venv(world: &mut E2eWorld) { + let python = interpreter_outside_a_venv(world); + std::fs::create_dir_all(python.parent().expect("interpreter has a folder")) + .expect("failed to create the interpreter's folder"); + rocm_core::fake_executable::write_fake_executable( + &python, + "#!/bin/sh\necho '{\"import_ok\": true}'\n", + ) + .expect("failed to write the stand-in interpreter"); +} + +#[when("the user adopts that interpreter without naming an environment folder")] +async fn user_adopts_interpreter_without_root(world: &mut E2eWorld) { + let python = interpreter_outside_a_venv(world) + .to_string_lossy() + .into_owned(); + let (stdout, stderr, rc) = crate::run_rocm(world, &["runtimes", "adopt", "--python", &python]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the refusal asks for the environment folder with --root")] +async fn assert_adopt_refusal_asks_for_root(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let combined = format!("{stdout}{stderr}"); + assert!( + combined.contains("could not infer the Python environment folder") + && combined.contains("pass --root"), + "the refusal does not ask for --root:\n{combined}" + ); +} + +#[then("no runtime is registered")] +async fn assert_no_runtime_registered(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["runtimes", "list"]); + assert!( + stdout.contains("installed: none") || stdout.contains("managed_runtimes: 0"), + "the refused adoption left a runtime registered:\n{stdout}" + ); +} From aedaa261777d12d02d8d6e5bce60aeff6b006d71 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 18:29:23 +0000 Subject: [PATCH 03/20] test(runtime): pin what each install-root payload check answers Two private helpers each decide whether an install root holds a payload: has_nontrivial_directory_contents in therock.rs, which refuses to extract a tarball into a non-empty target and decides whether a repair's replacement is already installed, and runtime_install_root_has_payload in runtime_registry.rs, which activation validation applies to a tarball runtime. Both skip dot-entries (the runtime marker is one) and count any other entry, a dangling symlink included. They differ only for a path that does not exist, or that is a dangling symlink: the first answers empty, the second returns an error. Add one table per helper over the same input kinds (missing path, empty directory, dotfile-only, dot-directory-only, regular file, subdirectory, a file as the root, dangling symlink entry, symlinked and dangling roots, unreadable directory) so the next commit, which collapses them into one, can be seen to change nothing but those two rows. Tests only. Signed-off-by: Jussi Elo --- apps/rocm/src/runtime_registry.rs | 100 ++++++++++++++++++++++++++++++ apps/rocm/src/therock.rs | 99 +++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 9f9fa17ce7..2f94a34448 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -946,4 +946,104 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// What `runtime_install_root_has_payload` answers for each kind of + /// install-root path: the same table `therock.rs` holds + /// `has_nontrivial_directory_contents` to, which differs in exactly the two + /// rows where the path does not exist. + #[test] + fn install_root_payload_characterisation() { + let base = tempfile::tempdir().expect("scratch dir"); + let mut seen = Vec::new(); + for (label, path) in install_root_cases(base.path()) { + let outcome = match runtime_install_root_has_payload(&path) { + Ok(true) => "payload", + Ok(false) => "empty", + Err(_) => "error", + }; + seen.push((label, outcome)); + } + let mut expected = vec![ + ("missing path", "error"), + ("empty dir", "empty"), + ("dotfile only", "empty"), + ("dot directory only", "empty"), + ("regular file", "payload"), + ("subdirectory", "payload"), + ("root is a regular file", "error"), + ]; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + expected.push(("dangling symlink entry", "payload")); + expected.push(("root is a symlink to a populated dir", "payload")); + expected.push(("root is a dangling symlink", "error")); + // Absent from `seen` when running as root, which reads a mode-000 + // directory anyway; then the case is neither built nor asserted. + if seen.iter().any(|(label, _)| *label == "unreadable dir") { + expected.push(("unreadable dir", "error")); + } + fs::set_permissions( + base.path().join("locked"), + fs::Permissions::from_mode(0o755), + ) + .ok(); + } + assert_eq!(sorted(seen), sorted(expected)); + } + + fn sorted(mut rows: Vec<(&'static str, &'static str)>) -> Vec<(&'static str, &'static str)> { + rows.sort_unstable(); + rows + } + + /// One path per input kind. An unreadable directory is only built when the + /// process cannot read it anyway (a root user can, and then the case says + /// nothing). + fn install_root_cases(base: &Path) -> Vec<(&'static str, PathBuf)> { + let mut cases = vec![("missing path", base.join("missing"))]; + let dir = |name: &str| { + let path = base.join(name); + fs::create_dir_all(&path).expect("create case dir"); + path + }; + cases.push(("empty dir", dir("empty"))); + let dotfile = dir("dotfile"); + fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + cases.push(("dotfile only", dotfile)); + let dotdir = dir("dotdir"); + fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); + cases.push(("dot directory only", dotdir)); + let file = dir("file"); + fs::write(file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("regular file", file)); + let sub = dir("sub"); + fs::create_dir_all(sub.join("bin")).expect("create subdir"); + cases.push(("subdirectory", sub)); + let not_a_dir = base.join("not-a-dir"); + fs::write(¬_a_dir, b"x").expect("write file root"); + cases.push(("root is a regular file", not_a_dir)); + #[cfg(unix)] + { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dangling = dir("dangling-entry"); + symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); + cases.push(("dangling symlink entry", dangling)); + let link_root = base.join("link-root"); + symlink(base.join("sub"), &link_root).expect("symlink root"); + cases.push(("root is a symlink to a populated dir", link_root)); + let dangling_root = base.join("dangling-root"); + symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); + cases.push(("root is a dangling symlink", dangling_root)); + let locked = dir("locked"); + fs::write(locked.join("payload.bin"), b"x").expect("write payload"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); + if fs::read_dir(&locked).is_err() { + cases.push(("unreadable dir", locked)); + } else { + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); + } + } + cases + } } diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 8efc17ebbe..2939fbd6d4 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -1708,4 +1708,103 @@ mod tests { "ROCm 7.14.0 is the newest version in this repository but has no installable PyTorch wheels for this Python and platform; installing ROCm 7.13.0 instead" ); } + + /// What `has_nontrivial_directory_contents` answers for each kind of + /// install-root path, so the sibling rule in `runtime_registry.rs` can be held + /// to the same table. + #[test] + fn nontrivial_directory_contents_characterisation() { + let base = tempfile::tempdir().expect("scratch dir"); + let mut seen = Vec::new(); + for (label, path) in install_root_cases(base.path()) { + let outcome = match has_nontrivial_directory_contents(&path) { + Ok(true) => "payload", + Ok(false) => "empty", + Err(_) => "error", + }; + seen.push((label, outcome)); + } + let mut expected = vec![ + ("missing path", "empty"), + ("empty dir", "empty"), + ("dotfile only", "empty"), + ("dot directory only", "empty"), + ("regular file", "payload"), + ("subdirectory", "payload"), + ("root is a regular file", "error"), + ]; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + expected.push(("dangling symlink entry", "payload")); + expected.push(("root is a symlink to a populated dir", "payload")); + expected.push(("root is a dangling symlink", "empty")); + // Absent from `seen` when running as root, which reads a mode-000 + // directory anyway; then the case is neither built nor asserted. + if seen.iter().any(|(label, _)| *label == "unreadable dir") { + expected.push(("unreadable dir", "error")); + } + fs::set_permissions( + base.path().join("locked"), + fs::Permissions::from_mode(0o755), + ) + .ok(); + } + assert_eq!(sorted(seen), sorted(expected)); + } + + fn sorted(mut rows: Vec<(&'static str, &'static str)>) -> Vec<(&'static str, &'static str)> { + rows.sort_unstable(); + rows + } + + /// One path per input kind. An unreadable directory is only built when the + /// process cannot read it anyway (a root user can, and then the case says + /// nothing). + fn install_root_cases(base: &Path) -> Vec<(&'static str, PathBuf)> { + let mut cases = vec![("missing path", base.join("missing"))]; + let dir = |name: &str| { + let path = base.join(name); + fs::create_dir_all(&path).expect("create case dir"); + path + }; + cases.push(("empty dir", dir("empty"))); + let dotfile = dir("dotfile"); + fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + cases.push(("dotfile only", dotfile)); + let dotdir = dir("dotdir"); + fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); + cases.push(("dot directory only", dotdir)); + let file = dir("file"); + fs::write(file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("regular file", file)); + let sub = dir("sub"); + fs::create_dir_all(sub.join("bin")).expect("create subdir"); + cases.push(("subdirectory", sub)); + let not_a_dir = base.join("not-a-dir"); + fs::write(¬_a_dir, b"x").expect("write file root"); + cases.push(("root is a regular file", not_a_dir)); + #[cfg(unix)] + { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dangling = dir("dangling-entry"); + symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); + cases.push(("dangling symlink entry", dangling)); + let link_root = base.join("link-root"); + symlink(base.join("sub"), &link_root).expect("symlink root"); + cases.push(("root is a symlink to a populated dir", link_root)); + let dangling_root = base.join("dangling-root"); + symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); + cases.push(("root is a dangling symlink", dangling_root)); + let locked = dir("locked"); + fs::write(locked.join("payload.bin"), b"x").expect("write payload"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); + if fs::read_dir(&locked).is_err() { + cases.push(("unreadable dir", locked)); + } else { + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); + } + } + cases + } } From 36a59a97af282db5ce47f042e4c9602f2770fb0a Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 18:43:26 +0000 Subject: [PATCH 04/20] test(runtimes): pin how the ComfyUI and engine paths pick the single ready runtime With no runtime chosen, `rocm comfyui install` and the engine paths (`serve`, `engines install`) each pick "the one ready runtime" with their own copy of the rule. Both count a runtime as ready by `validate_runtime_manifest_for_activation`, but they differ in what happens around it: the engine path loads config and runs `recover_setup_runtime_registration` first and answers no selection on zero or many (its callers print their own advice), while the ComfyUI path never recovers and refuses on zero or many with its own text. Pin both as they are today, for zero, one and several ready runtimes and for a setup runtime the registry has lost. Each test that checks a refusal also checks the state it describes: that nothing was written, and that the remedy the message names (`rocm runtimes activate`, `--runtime-id`, repairing the runtime) clears it. No production code changes. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 157 +++++++++++++++++++++++++ apps/rocm/src/engine_runtime_select.rs | 91 ++++++++++++++ apps/rocm/src/engines_cmd.rs | 68 +++++++++++ apps/rocm/src/test_support.rs | 37 ++++++ 4 files changed, 353 insertions(+) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 3e496900be..f2b1a1a815 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -397,6 +397,7 @@ mod tests { use super::*; use crate::comfyui::render_status; use crate::comfyui::tests::{ready_runtime_manifest, test_paths, write_runtime_manifest}; + use crate::test_support::{runtime_registry_entry_names, write_test_pip_runtime}; use rocm_core::{runtime_python_executable_in_env, runtime_rocm_library_filename}; use std::fs; @@ -958,6 +959,162 @@ mod tests { Ok(()) } + /// Zero runtimes registered: the refusal points at the launcher's setup row, + /// and selecting wrote nothing. + #[test] + fn comfyui_selection_with_no_registered_runtime_says_to_install_rocm_and_writes_nothing() + -> Result<()> { + let paths = test_paths("comfyui-selection-none-registered"); + + let message = select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("no registered runtime means no selection") + .to_string(); + + assert!( + message.contains("Install ROCm first") + && message.contains("pick \"Set up this system\""), + "unexpected refusal: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + + /// One ready runtime and nothing chosen: it is the selection. + #[test] + fn comfyui_selection_with_one_ready_runtime_selects_it() -> Result<()> { + let paths = test_paths("comfyui-selection-one-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + + let selected = select_runtime(&paths, &RocmCliConfig::default(), None)?; + + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + Some(selected.python.display().to_string()), + manifest.python_executable + ); + Ok(()) + } + + /// Registered but none ready: the refusal lists the runtime with the reason + /// it is unusable, and fixing that reason (here: putting the interpreter + /// back) is what clears it. + #[test] + fn comfyui_selection_with_only_an_unusable_runtime_clears_once_it_is_fixed() -> Result<()> { + let paths = test_paths("comfyui-selection-none-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + ); + fs::remove_file(&python)?; + + let message = select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("an unusable runtime is not ready") + .to_string(); + + assert!( + message.contains("No ROCm runtime is ready") + && message.contains("rocm runtimes list") + && message.contains(&format!("`{}` (unusable", manifest.runtime_key)), + "unexpected refusal: {message}" + ); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + + fs::write(&python, "python")?; + let selected = select_runtime(&paths, &RocmCliConfig::default(), None)?; + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + Ok(()) + } + + /// Several ready runtimes and none chosen: the refusal names the remedies, + /// and each one clears it. `rocm runtimes activate ` is run as + /// `activate_runtime`, its implementation, and `--runtime-id` as the selector. + #[test] + fn comfyui_selection_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let paths = test_paths("comfyui-selection-several-ready"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let second = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = select_runtime(&paths, &config, None) + .expect_err("several ready runtimes and no choice must not select") + .to_string(); + + assert!( + message.contains("Multiple ROCm runtimes are ready") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id ") + && message.contains(&first.runtime_key) + && message.contains(&second.runtime_key), + "unexpected refusal: {message}" + ); + + let selected = select_runtime(&paths, &config, Some(&first.runtime_key))?; + assert_eq!(selected.manifest.runtime_key, first.runtime_key); + + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &second.runtime_key)?; + let selected = select_runtime(&paths, &config, None)?; + assert_eq!(selected.manifest.runtime_key, second.runtime_key); + Ok(()) + } + + /// The setup-recovery half of the rule, today's behaviour: a ready runtime + /// that only `recover_setup_runtime_registration` can make visible is not + /// found by the ComfyUI path. It reports an empty registry and writes + /// nothing, where the engine path (see `engine_runtime_select.rs`'s + /// `engine_selection_recovers_the_setup_runtime_the_registry_lost`) selects it. + #[test] + fn comfyui_selection_does_not_recover_a_setup_runtime_the_registry_lost() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-recovery"); + crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!( + config.setup.therock_venv.is_some(), + "setup must name the folder for this test to mean anything" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + + let message = select_runtime(&paths, &config, None) + .expect_err("the registry is empty and ComfyUI does not recover") + .to_string(); + + assert!(message.contains("Install ROCm first"), "got: {message}"); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + fn command_env_value(command: &Command, key: &str) -> Option { command .get_envs() diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 4846f5425b..73de340733 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -1048,4 +1048,95 @@ mod tests { let _ = std::fs::remove_dir_all(root); Ok(()) } + + /// Zero ready runtimes, the registry empty: the engine path answers with no + /// selection rather than an error, and writes nothing. The caller reports + /// the missing runtime in its own words (`engines_cmd.rs` and `serve_cmd.rs`). + #[test] + fn engine_selection_with_no_registered_runtime_stays_unselected_and_writes_nothing() + -> Result<()> { + let (root, paths) = test_paths("no-registered-runtime-selection"); + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.env_id.is_none()); + assert!(selection.source.is_none()); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Zero ready runtimes, one registered but unusable: still no selection. + /// Readiness is `validate_runtime_manifest_for_activation`, so a runtime + /// whose interpreter is gone does not count, and it is not picked as "the + /// only one" either. + #[test] + fn engine_selection_does_not_count_an_unusable_runtime_as_ready() -> Result<()> { + let (root, paths) = test_paths("unusable-runtime-selection"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = manifest + .python_executable + .as_deref() + .expect("the fixture records an interpreter"); + std::fs::remove_file(python)?; + assert!(validate_runtime_manifest_for_activation(&manifest).is_err()); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.source.is_none()); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// engine path, and finding it re-files its registry entry, so the message + /// and the state are one fact (the selection names a key that + /// `rocm runtimes list` now shows). + #[test] + fn engine_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { + let (root, paths) = test_paths("setup-recovery-selection"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "the registry must start without the runtime for this test to mean anything" + ); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert_eq!( + selection.runtime_id.as_deref(), + Some(manifest.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 7f1ecbc835..cef764ffd0 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -834,4 +834,72 @@ mod tests { ); } } + + /// Several ready runtimes and none chosen: the engine path does not guess. + /// `resolve_engine_env` reports the gap in its own words, and each remedy + /// those words name must clear it: `rocm runtimes activate ` + /// (run here as `activate_runtime`, its implementation) and `--runtime-id`. + #[test] + fn engine_install_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let (root, paths) = test_paths("engine-install-several-ready-runtimes"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = resolve_engine_env(&paths, &config, "vllm", None, None) + .expect_err("several ready runtimes and no choice must not resolve") + .to_string(); + assert!( + message.contains("no active ROCm runtime is configured") + && message.contains("rocm runtimes list") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id"), + "unexpected refusal: {message}" + ); + + // `rocm runtimes activate `: the choice is now made. + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &first.runtime_key)?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&config, "vllm", None, None), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); + + // `--runtime-id `: the same choice, for one invocation. + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection( + &RocmCliConfig::default(), + "vllm", + Some(&first.runtime_key), + None, + ), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("cli_runtime_id")); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index c724ad973f..9cbb22aa61 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -229,6 +229,43 @@ pub(crate) fn write_test_pip_runtime( Ok(manifest) } +/// Leave `paths` as `rocm setup` followed by a lost registry entry would: a ready +/// wheel runtime whose install tree holds its own `.rocm-cli-runtime.json` and +/// that `config.json` names as the setup folder, but that the runtime registry +/// has no entry for. Only `recover_setup_runtime_registration` can make it +/// visible again, so a selection that finds it has run that recovery. +pub(crate) fn write_test_setup_runtime_missing_from_registry( + paths: &AppPaths, + runtime_key: &str, +) -> Result { + let manifest = write_test_pip_runtime( + paths, + runtime_key, + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(runtime_manifest_path(paths, runtime_key))?; + let mut config = rocm_core::RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + config.save(paths)?; + Ok(manifest) +} + +/// File names in the runtime registry directory, sorted; empty when the +/// directory does not exist. Lets a test assert that a selection did, or did +/// not, write a registry entry. +pub(crate) fn runtime_registry_entry_names(paths: &AppPaths) -> Vec { + let mut names: Vec = fs::read_dir(runtime_registry_dir(paths)) + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + mod tests { use super::*; From b8331b307ec9a074a8ed509823d879e230fa58bc Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 19:19:37 +0000 Subject: [PATCH 05/20] fix(comfyui): pick the single ready runtime by the same rule as the engine paths With no runtime chosen, `rocm comfyui install` and the engine paths (`rocm serve`, `rocm engines install`) each kept their own copy of "use the one ready runtime". They agreed on what ready means but not on the steps around it: the engine path recovered the runtime `rocm setup` created before counting, ComfyUI never did, so a runtime whose registry entry had been lost made the install refuse with "Install ROCm first" (or silently pick a different runtime registered beside it) while the same machine served fine. Move the rule to `runtime_adopt::select_single_ready_runtime`, beside `recover_setup_runtime_registration`: recover setup, load the registry, count the runtimes `validate_runtime_manifest_for_activation` accepts, and return one, none (with everything registered) or several (the ready ones). Both callers use it and delete their copy. Each keeps its own words: the engine path still treats none and several as "no selection" and leaves the advice to its callers, and ComfyUI still refuses with its own `/runtimes`, `rocm runtimes activate` and `--runtime-id` text. What changes for a ComfyUI user, only when no `--runtime-id`, active runtime or configured default says which runtime to use: the runtime setup created is found even if its registry entry was lost, and finding it writes the entry back (also under `--dry-run`); with another ready runtime registered beside it the install now refuses as ambiguous instead of picking the other one; and an unusable setup manifest now fails the install with the recovery error instead of being skipped. The engine paths behave as before. The ComfyUI characterisation test for recovery flips to the new behaviour, and a test now runs both callers against the same registry states and fails if they disagree. The e2e scenario comfyui-15 covers the recovery through the dry run. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 244 +++++++++++++----- apps/rocm/src/engine_runtime_select.rs | 26 +- apps/rocm/src/runtime_adopt.rs | 147 ++++++++++- docs/architecture.md | 6 +- docs/testing.md | 9 + tests/e2e-cucumber/features/comfyui.feature | 16 ++ tests/e2e-cucumber/tests/e2e/comfyui_steps.rs | 109 +++++++- 7 files changed, 467 insertions(+), 90 deletions(-) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index f2b1a1a815..fc6b2689f4 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -17,6 +17,7 @@ //! ComfyUI types. use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; +use crate::runtime_adopt::{SingleReadyRuntime, select_single_ready_runtime}; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; use anyhow::{Context, Result, bail}; @@ -33,15 +34,11 @@ pub(super) fn select_runtime( config: &RocmCliConfig, selector: Option<&str>, ) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - if manifests.is_empty() { - bail!( - "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." - ); - } let manifest = match selector.map(str::trim).filter(|value| !value.is_empty()) { - Some(selector) => select_runtime_by_selector(&manifests, selector)?.clone(), - None => select_default_runtime(config, &manifests)?.clone(), + Some(selector) => { + select_runtime_by_selector(®istered_runtimes(paths)?, selector)?.clone() + } + None => select_default_runtime(paths, config)?, }; if runtime_usability_status(&manifest) != "ready" { bail!( @@ -119,16 +116,41 @@ fn format_runtime_statuses<'a>( .join(", ") } -fn select_default_runtime<'a>( +/// Every registered runtime, or the refusal to install ROCm when there is none. +fn registered_runtimes(paths: &AppPaths) -> Result> { + let manifests = therock::load_runtime_manifests(paths)?; + if manifests.is_empty() { + return Err(install_rocm_first()); + } + Ok(manifests) +} + +fn install_rocm_first() -> anyhow::Error { + anyhow::anyhow!( + "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." + ) +} + +fn select_default_runtime( + paths: &AppPaths, config: &RocmCliConfig, - manifests: &'a [therock::InstalledRuntimeManifest], -) -> Result<&'a therock::InstalledRuntimeManifest> { +) -> Result { if let Some(active_key) = config.active_runtime_key.as_deref() { - return select_runtime_by_selector(manifests, active_key); + return select_runtime_by_selector(®istered_runtimes(paths)?, active_key).cloned(); } - let Some(default_runtime_id) = config.default_runtime_id.as_deref() else { - return select_single_ready_runtime(manifests); - }; + match config.default_runtime_id.as_deref() { + Some(default_runtime_id) => { + select_runtime_by_default_id(®istered_runtimes(paths)?, default_runtime_id).cloned() + } + // Nothing says which runtime to use: the rule the engine paths share. + None => single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config)?), + } +} + +fn select_runtime_by_default_id<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + default_runtime_id: &str, +) -> Result<&'a therock::InstalledRuntimeManifest> { let matches = manifests .iter() .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(default_runtime_id)) @@ -156,23 +178,24 @@ fn select_default_runtime<'a>( } } -fn select_single_ready_runtime( - manifests: &[therock::InstalledRuntimeManifest], -) -> Result<&therock::InstalledRuntimeManifest> { - let ready = manifests - .iter() - .filter(|manifest| runtime_usability_status(manifest) == "ready") - .collect::>(); - match ready.as_slice() { - [manifest] => Ok(*manifest), - [] => { - let statuses = format_runtime_statuses(manifests); +/// ComfyUI's words for what [`select_single_ready_runtime`] found. The rule +/// itself, including which runtimes count as ready, is not decided here. +fn single_ready_runtime_or_refusal( + found: SingleReadyRuntime, +) -> Result { + match found { + SingleReadyRuntime::One(manifest) => Ok(*manifest), + SingleReadyRuntime::Zero { registered } if registered.is_empty() => { + Err(install_rocm_first()) + } + SingleReadyRuntime::Zero { registered } => { + let statuses = format_runtime_statuses(®istered); bail!( "No ROCm runtime is ready. Fix one or install another: run `rocm` and pick \"Set up this system\" (or run `rocm install sdk`), then install ComfyUI. Runtimes found: {statuses}. See `rocm runtimes list`." ) } - _ => { - let available = format_available_runtime_keys(ready.iter().copied()); + SingleReadyRuntime::Many { ready } => { + let available = format_available_runtime_keys(&ready); bail!( "Multiple ROCm runtimes are ready. Pick one in `/runtimes`, set a default with `rocm runtimes activate `, or pass `--runtime-id `. Available: {available}. See `rocm runtimes list`." ) @@ -397,6 +420,7 @@ mod tests { use super::*; use crate::comfyui::render_status; use crate::comfyui::tests::{ready_runtime_manifest, test_paths, write_runtime_manifest}; + use crate::runtime_adopt::classify_single_ready_runtime; use crate::test_support::{runtime_registry_entry_names, write_test_pip_runtime}; use rocm_core::{runtime_python_executable_in_env, runtime_rocm_library_filename}; use std::fs; @@ -462,7 +486,8 @@ mod tests { }; let manifests = [manifest]; - let selected = select_default_runtime(&RocmCliConfig::default(), &manifests)?; + let selected = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec()))?; assert_eq!( selected.runtime_key, "release-pip-gfx120x-all-7-13-0a20260511" @@ -511,8 +536,9 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("two ready runtimes should be ambiguous"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("two ready runtimes should be ambiguous"); let message = error.to_string(); assert_actionable(&message); @@ -546,8 +572,9 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; let manifests = [unusable]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("an unusable runtime is not ready"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("an unusable runtime is not ready"); let message = error.to_string(); assert!( @@ -654,11 +681,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("no-such-runtime-id".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "no-such-runtime-id") .expect_err("configured default id matches nothing"); let message = error.to_string(); @@ -690,11 +713,7 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -722,11 +741,7 @@ mod tests { ); let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -1087,15 +1102,14 @@ mod tests { Ok(()) } - /// The setup-recovery half of the rule, today's behaviour: a ready runtime - /// that only `recover_setup_runtime_registration` can make visible is not - /// found by the ComfyUI path. It reports an empty registry and writes - /// nothing, where the engine path (see `engine_runtime_select.rs`'s - /// `engine_selection_recovers_the_setup_runtime_the_registry_lost`) selects it. + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// ComfyUI path, as it is by the engine paths, and finding it re-files its + /// registry entry, so the selection names a key `rocm runtimes list` shows. #[test] - fn comfyui_selection_does_not_recover_a_setup_runtime_the_registry_lost() -> Result<()> { + fn comfyui_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { let paths = test_paths("comfyui-selection-setup-recovery"); - crate::test_support::write_test_setup_runtime_missing_from_registry( + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( &paths, "release-pip-gfx120x-all-7-14-0", )?; @@ -1106,12 +1120,122 @@ mod tests { ); assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); - let message = select_runtime(&paths, &config, None) - .expect_err("the registry is empty and ComfyUI does not recover") - .to_string(); + let selected = select_runtime(&paths, &config, None)?; - assert!(message.contains("Install ROCm first"), "got: {message}"); - assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + Ok(()) + } + + /// The two paths that pick "the one ready runtime" must not drift apart: + /// for each registry state below, ComfyUI selects a runtime exactly when + /// the engine paths do, and the same one. Both reach the rule through + /// `select_single_ready_runtime`, so a copy of it re-introduced in either + /// caller (a different readiness test, a missing setup recovery) shows up as + /// a disagreement on one of these states. + #[test] + fn comfyui_and_engine_paths_agree_on_the_single_ready_runtime() -> Result<()> { + use crate::engine_runtime_select::{ + resolve_engine_selection, validate_engine_selection_runtime, + }; + + #[derive(Clone, Copy)] + enum State { + Empty, + OneUnusable, + OneReady, + TwoReady, + ReadyAndUnusable, + SetupRuntimeLost, + SetupRuntimeLostBesideAReadyOne, + } + for (label, state, expected) in [ + ("empty", State::Empty, None), + ("one-unusable", State::OneUnusable, None), + ("one-ready", State::OneReady, Some("runtime-a")), + ("two-ready", State::TwoReady, None), + ( + "ready-and-unusable", + State::ReadyAndUnusable, + Some("runtime-a"), + ), + ( + "setup-runtime-lost", + State::SetupRuntimeLost, + Some("runtime-s"), + ), + ( + "setup-runtime-lost-beside-a-ready-one", + State::SetupRuntimeLostBesideAReadyOne, + None, + ), + ] { + // Each path gets its own copy of the state: a recovery run by one + // writes the registry, which would otherwise hand the other path a + // state it could not have found for itself. + let plant_state = |paths: &AppPaths| -> Result<()> { + let plant = |key: &str| { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + }; + let break_runtime = |manifest: &therock::InstalledRuntimeManifest| -> Result<()> { + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + Ok(()) + }; + match state { + State::Empty => {} + State::OneUnusable => break_runtime(&plant("runtime-a")?)?, + State::OneReady => { + plant("runtime-a")?; + } + State::TwoReady => { + plant("runtime-a")?; + plant("runtime-b")?; + } + State::ReadyAndUnusable => { + plant("runtime-a")?; + break_runtime(&plant("runtime-b")?)?; + } + State::SetupRuntimeLost => { + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + State::SetupRuntimeLostBesideAReadyOne => { + plant("runtime-a")?; + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + } + Ok(()) + }; + let comfyui_paths = test_paths(&format!("comfyui-engine-agree-{label}-comfyui")); + plant_state(&comfyui_paths)?; + let comfyui = + select_runtime(&comfyui_paths, &RocmCliConfig::load(&comfyui_paths)?, None) + .ok() + .map(|selected| selected.manifest.runtime_key); + let engine_paths = test_paths(&format!("comfyui-engine-agree-{label}-engine")); + plant_state(&engine_paths)?; + let engine = validate_engine_selection_runtime( + &engine_paths, + resolve_engine_selection(&RocmCliConfig::load(&engine_paths)?, "vllm", None, None), + )? + .runtime_id; + + assert_eq!(comfyui, engine, "{label}: the two paths disagree"); + assert_eq!(comfyui.as_deref(), expected, "{label}"); + } Ok(()) } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 73de340733..6e04cfa53f 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -57,7 +57,10 @@ //! `resolve_runtime_selector_to_exact_key` is also `pub(crate)`: `engines_cmd.rs` //! calls it in production, and `runtimes_cmd.rs`'s own test module calls it //! directly too. `single_ready_runtime_key` is called only from -//! `validate_engine_selection_runtime` in this same module and stays private. +//! `validate_engine_selection_runtime` in this same module and stays private; +//! it does not decide which runtime counts as "the one ready runtime" itself, +//! it asks `runtime_adopt::select_single_ready_runtime`, the rule +//! `rocm comfyui install` shares. //! `optional_arg` and //! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read @@ -103,10 +106,8 @@ use rocm_core::{ }; use rocm_engine_protocol::EngineRecipeHint; -use crate::runtimes_cmd::{ - recover_setup_runtime_registration, select_runtime_manifest, - validate_runtime_manifest_for_activation, -}; +use crate::runtime_adopt::{SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtimes_cmd::{recover_setup_runtime_registration, select_runtime_manifest}; use crate::therock; #[derive(Debug, Clone)] @@ -211,15 +212,11 @@ pub(crate) fn validate_engine_selection_runtime( fn single_ready_runtime_key(paths: &AppPaths) -> Result> { let config = RocmCliConfig::load(paths).unwrap_or_default(); - recover_setup_runtime_registration(paths, &config)?; - let manifests = therock::load_runtime_manifests(paths)?; - let ready = manifests - .iter() - .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) - .collect::>(); - Ok(match ready.as_slice() { - [manifest] => Some(manifest.runtime_key.clone()), - _ => None, + // Zero or several ready runtimes leave the selection open: the caller that + // needs a runtime says so in its own words. + Ok(match select_single_ready_runtime(paths, &config)? { + SingleReadyRuntime::One(manifest) => Some(manifest.runtime_key), + SingleReadyRuntime::Zero { .. } | SingleReadyRuntime::Many { .. } => None, }) } @@ -611,6 +608,7 @@ mod tests { use serde_json::json; use super::*; + use crate::runtime_registry::validate_runtime_manifest_for_activation; use crate::status_for_readiness; use crate::test_support::write_test_pip_runtime; use crate::tests::{test_app_paths, test_paths}; diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 549753818a..60862770dc 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -5,9 +5,13 @@ //! Registering a runtime that was not installed by `rocm install sdk`: //! importing a manifest written elsewhere, adopting an existing Python //! environment by probing it, and recovering the registration of the runtime -//! `rocm setup` created. +//! `rocm setup` created. It also holds the one rule for picking "the one ready +//! runtime" when nothing else says which to use +//! ([`select_single_ready_runtime`]), which sits here because it recovers the +//! setup registration first; `rocm comfyui install` and the engine paths both +//! ask it and each words the zero and several cases itself. //! -//! Relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` +//! The rest was relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` //! calls into this module for `rocm runtimes import` and `rocm runtimes adopt`, //! and re-exports `newest_installed_runtime_manifest`, //! `recover_setup_runtime_registration` and `setup_runtime_paths` for the @@ -436,6 +440,64 @@ pub(crate) fn recover_setup_runtime_registration( Ok(Some(manifest.runtime_key)) } +/// What picking "the one ready runtime" found. The caller decides how to report +/// the two cases that are not a pick, because the advice differs by command +/// (`rocm comfyui install` and `rocm serve` name different remedies). +#[derive(Debug)] +pub(crate) enum SingleReadyRuntime { + /// Exactly one registered runtime is ready. + One(Box), + /// No registered runtime is ready. `registered` is everything the registry + /// holds, unusable entries included and possibly none, so a caller can say + /// why each one does not count. + Zero { + registered: Vec, + }, + /// More than one is ready, so there is nothing to pick without a choice. + Many { + ready: Vec, + }, +} + +/// Pick the one ready runtime when nothing else says which to use: no +/// `--runtime-id`, no active runtime, no configured default. This is the only +/// place that rule lives; `rocm comfyui install` and the engine paths +/// (`rocm serve`, `rocm engines install`) both call it. +/// +/// Ready means `validate_runtime_manifest_for_activation` accepts the runtime, +/// the same check `rocm runtimes activate` applies. Setup is recovered first +/// (`recover_setup_runtime_registration`), so the runtime `rocm setup` created +/// counts even if its registry entry was lost, and recovering it writes that +/// entry back. +pub(crate) fn select_single_ready_runtime( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result { + recover_setup_runtime_registration(paths, config)?; + Ok(classify_single_ready_runtime( + therock::load_runtime_manifests(paths)?, + )) +} + +/// The pure half of [`select_single_ready_runtime`]: which of `registered` are +/// ready, and whether that is none, one or several. +pub(crate) fn classify_single_ready_runtime( + registered: Vec, +) -> SingleReadyRuntime { + let ready: Vec<_> = registered + .iter() + .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) + .cloned() + .collect(); + if ready.is_empty() { + return SingleReadyRuntime::Zero { registered }; + } + match <[therock::InstalledRuntimeManifest; 1]>::try_from(ready) { + Ok([manifest]) => SingleReadyRuntime::One(Box::new(manifest)), + Err(ready) => SingleReadyRuntime::Many { ready }, + } +} + /// The folder setup was configured for and its pip cache — the pair both /// `examine` forms report. Shared so the two cannot name different folders. /// @@ -819,4 +881,85 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + fn pip_runtime(paths: &AppPaths, key: &str) -> Result { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + } + + /// Readiness is the activation check and nothing looser: a runtime whose + /// interpreter is gone is registered but does not count, and neither is it + /// "the only one". + #[test] + fn single_ready_runtime_counts_only_what_activation_accepts() -> Result<()> { + let (root, paths) = test_paths("single-ready-classify"); + let ready = pip_runtime(&paths, "runtime-a")?; + let broken = pip_runtime(&paths, "runtime-b")?; + fs::remove_file( + broken + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + assert!(validate_runtime_manifest_for_activation(&ready).is_ok()); + assert!(validate_runtime_manifest_for_activation(&broken).is_err()); + + match classify_single_ready_runtime(vec![ready.clone(), broken.clone()]) { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-a"), + other => panic!("expected the one ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(vec![broken.clone()]) { + SingleReadyRuntime::Zero { registered } => { + assert_eq!(registered.len(), 1, "unusable entries stay listed"); + } + other => panic!("expected no ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(Vec::new()) { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + let second = pip_runtime(&paths, "runtime-c")?; + match classify_single_ready_runtime(vec![ready, broken, second]) { + SingleReadyRuntime::Many { ready } => { + let keys: Vec<_> = ready.iter().map(|m| m.runtime_key.as_str()).collect(); + assert_eq!(keys, ["runtime-a", "runtime-c"], "only ready ones listed"); + } + other => panic!("expected several ready runtimes, got {other:?}"), + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The selection recovers the setup runtime first, so a runtime the registry + /// lost is the one found, and finding it is what writes its entry back. + #[test] + fn select_single_ready_runtime_recovers_setup_before_counting() -> Result<()> { + let (root, paths) = test_paths("single-ready-recovers-setup"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + match select_single_ready_runtime(&paths, &config)? { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-s"), + other => panic!("expected the recovered setup runtime, got {other:?}"), + } + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", lost.runtime_key)] + ); + + // Without setup naming a folder there is nothing to recover and the + // registry is not touched. + let (other_root, other_paths) = test_paths("single-ready-no-setup"); + match select_single_ready_runtime(&other_paths, &RocmCliConfig::default())? { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + assert!(crate::test_support::runtime_registry_entry_names(&other_paths).is_empty()); + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(other_root); + Ok(()) + } } diff --git a/docs/architecture.md b/docs/architecture.md index e6bb942d19..dc51b85ae6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -49,7 +49,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. - [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). Two items that sit physically inside this span in `main.rs` stayed there instead of moving: `missing_packaged_engine_reason` has callers outside the cluster (plus its own test), and `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request`. `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation (Phase 6f, ROCMAI-208, folding Phase 6f/ROCMAI-86 closed): the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`), the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`), and the model-catalog rendering behind `rocm model` (alias `models`) and `rocm model --verbose` (`model_catalog_platform_heading`, `render_model_registry_text_with_context_and_host`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`, `render_model_registry_verbose_text_with_context_and_host`, `model_catalog_header`, `append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `format_list_or_none`, `format_string_map_or_none`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `format_bytes`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, `format_gib`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`, `engine_runtime_is_native_windows_unsupported`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside this cluster: the `DEFAULT_ENGINE_MARKER` constant (shared by both engine-inventory renderers so the two copies cannot drift apart) and `model_registry_adapter_availability_note` (read only from `append_model_engine_support_lines` and this module's own test); both stay private. `render_config_text` and `telemetry_policy_summary` sat immediately adjacent to this cluster's end boundary in `main.rs` and were explicitly out of scope, so that split left them where they were; they now live in [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs). `dispatch()` calls directly into this module at two points — `render_model_registry_verbose_text_with_context_and_host` and `render_model_registry_text_with_context_and_host` — re-imported via `use crate::examine_report_text::{...};`; [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report` itself. `main.rs`'s own `launch_default` calls `render_launch_summary` directly, also re-imported there. `render_examine_text`, `render_examine_text_with_paths`, and `render_model_registry_verbose_text_with_context_and_host` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `render_engine_inventory_text_with_paths` is also read from `chat.rs` and from `engines_cmd.rs`'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build; `format_bytes` is also `pub(crate)`, read from [`storage.rs`](../apps/rocm/src/storage.rs)'s cache-size reporting and [`automations.rs`](../apps/rocm/src/automations.rs)'s plain-language summaries. Every other item in this list has no caller outside this module and stays private. This cluster also reaches three items that stayed behind in `main.rs` because they already had callers outside it: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)), `engine_ruled_out_by_platform` (also read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs)'s `assess_model_for_host`; it has since moved there and is re-imported in `main.rs`), and `missing_packaged_engine_reason` (already noted in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s entry above as having callers outside that cluster, plus its own test — this cluster is one of them) — all three reached via `use crate::{...};`. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`, the same way other mechanically relocated modules do. Two tests interleaved among this cluster's tests in `main.rs`'s old shared `mod tests` stayed behind because their primary subject is unrelated: `missing_packaged_engine_reason`'s own test and the whole-command-tree ASCII-help walk (two CLI-lifecycle-audit-log tests that were also left there have since moved to [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs)). @@ -68,7 +68,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. - [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. - [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) — the argv helpers that `chat.rs`, `tool_policy.rs` and the freeform planner share, in a leaf module whose only `crate::` dependency is the leaf [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs): `chat_cli_arg_value`, `chat_cli_arg_value_checked`, `chat_cli_has_flag` and `ensure_flag` (from `chat.rs`), and `format_structured_tool_call` (which quotes each argument with `shell_quote`), `serve_args_request_cpu_device` and `device_policy_is_cpu_mode` (from `freeform_planner.rs`). They moved so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `ensure_flag` (read by `freeform_execution.rs`) and `freeform_planner.rs` re-exports `format_structured_tool_call`, so every other caller keeps its path. -- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. +- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. @@ -112,7 +112,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: - [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). - [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. - [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. +- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. - [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. - [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. diff --git a/docs/testing.md b/docs/testing.md index 97aa511110..39044fe274 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -780,6 +780,15 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. +`comfyui-15` covers which runtime `rocm comfyui install` uses when none is +chosen: it recovers the runtime `rocm setup` created before counting the ready +ones, as `rocm serve` does, so a runtime whose registry entry was lost is found +and its entry written back. It runs with no GPU, through the install's dry run: + +```bash +cargo xtask e2e -- -n "comfyui-15" +``` + `comfyui-14` is different: it installs ComfyUI into a real, isolated managed runtime and checks the ROCm torch stack survives with no `nvidia-*` packages added. It carries `@requires-gpu @nightly @requires-os:linux @requires-bare-metal` diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index 036d9615a5..1413cadefb 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -328,3 +328,19 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv And ComfyUI's dependencies were installed into the runtime And the runtime's torch stack is unchanged And no nvidia-* packages were added to the runtime + + # With no runtime chosen, `rocm comfyui install` uses the one ready runtime, + # and it finds it the way `rocm serve` and `rocm engines install` do: it + # recovers the runtime `rocm setup` created first. A runtime whose registry + # entry was lost, but whose install tree still holds its own manifest and that + # setup names, therefore counts as the ready runtime and its entry is written + # back, instead of the install refusing with "Install ROCm first" while setup + # has finished. The dry run is the cheapest way to see which runtime the + # install chose: it prints the `--runtime-id` of the command it would run. + # No GPU is needed: readiness is filesystem and manifest state. + @id:comfyui-install-finds-the-runtime-setup-created + Scenario: comfyui-15 - ComfyUI install finds the runtime setup created when the registry lost its entry + Given a ready ROCm runtime that setup created and the registry has forgotten + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects that runtime + And the runtime registry has an entry for it again diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index 7fe33a9917..edf3cba9cc 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,6 +34,13 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! +//! `comfyui-15` covers the other half of that selection: with nothing chosen, +//! the install recovers the runtime `rocm setup` created before it counts the +//! ready ones, as `rocm serve` does, so a runtime the registry lost is found and +//! its entry written back. It plants a ready runtime and the setup folder that +//! names it, but no registry entry, and previews the install with `--dry-run`, +//! which prints the `--runtime-id` of the runtime it chose. +//! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for //! `download_and_extract_source`'s in-process `GzDecoder`/`tar` unpack, which @@ -194,6 +201,21 @@ fn rocm_library_filename(shortname: &str) -> String { /// confusing downstream assertion instead of on the fixture. If this step starts /// failing after an edit here, suspect the manifest shape first. fn plant_ready_runtime(data: &Path, key: &str) { + let manifest = plant_ready_runtime_tree(data, key); + let registry = data.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), + ) + .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); +} + +/// The on-disk half of [`plant_ready_runtime`]: the install root and what the +/// readiness check needs under it. Returns the manifest JSON that describes it, +/// leaving it to the caller whether the registry gets an entry. +fn plant_ready_runtime_tree(data: &Path, key: &str) -> serde_json::Value { let install_root = data.join("runtimes").join("roots").join(key); let sdk_root = install_root.join("sdk"); let sdk_bin = sdk_root.join("bin"); @@ -208,7 +230,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { write_fixture(&amdhip, "stub"); write_fixture(&hipblas, "stub"); - let manifest = serde_json::json!({ + serde_json::json!({ "runtime_key": key, "runtime_id": key, "channel": "release", @@ -229,16 +251,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { ], }, "installed_at_unix_ms": 1, - }); - - let registry = data.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); - std::fs::write( - registry.join(format!("{key}.json")), - serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), - ) - .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); + }) } /// Combined stdout+stderr of the recorded `rocm` invocation. The refusal is an @@ -536,6 +549,80 @@ async fn refusal_lists_both_keys(world: &mut E2eWorld) { } } +// --- comfyui-15: the install finds the runtime setup created --- + +/// The one runtime `comfyui-15` plants. Not in [`RUNTIME_KEYS`], which are the +/// two the ambiguity scenario needs. +const SETUP_RUNTIME_KEY: &str = "e2e-setup-created-runtime"; + +fn registry_dir(world: &E2eWorld) -> PathBuf { + data_dir(world).join("runtimes").join("registry") +} + +#[given("a ready ROCm runtime that setup created and the registry has forgotten")] +async fn plant_runtime_setup_created_and_registry_forgot(world: &mut E2eWorld) { + let data = data_dir(world); + let manifest = plant_ready_runtime_tree(&data, SETUP_RUNTIME_KEY); + let install_root = data.join("runtimes").join("roots").join(SETUP_RUNTIME_KEY); + // The install tree's own copy of its manifest is what makes the registry + // entry recoverable. Without it recovery finds nothing to restore and the + // scenario would pass or fail for the wrong reason. + write_fixture( + &install_root.join(".rocm-cli-runtime.json"), + &serde_json::to_string_pretty(&manifest).expect("manifest serialises"), + ); + // Black-box: plain JSON matching the CLI's on-disk config schema. `setup` + // naming the folder is what lets recovery find the tree; there is no + // registry entry, and that is the state under test. + write_fixture( + &root(world).join("config").join("config.json"), + &serde_json::to_string_pretty(&serde_json::json!({ + "setup": { "therock_venv": install_root.display().to_string() }, + })) + .expect("config serialises"), + ); + assert!( + !registry_dir(world) + .join(format!("{SETUP_RUNTIME_KEY}.json")) + .exists(), + "the registry must start without the runtime for this scenario to mean anything" + ); +} + +#[when("the user previews a ComfyUI install without choosing a runtime")] +async fn preview_comfyui_install_without_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["comfyui", "install", "--dry-run"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the preview selects that runtime")] +async fn preview_selects_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the preview should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.lines().any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {SETUP_RUNTIME_KEY}"))), + "the preview should name the runtime setup created in its install command, got:\n{text}" + ); +} + +#[then("the runtime registry has an entry for it again")] +async fn registry_has_the_setup_runtime_again(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + entry.is_file(), + "selecting the runtime setup created should have written {} back", + entry.display() + ); +} + /// Pacing knobs for `comfyui-04`'s download server, mirroring /// `therock_steps.rs`'s `PACED_TARBALL_*` constants: large enough that several /// chunk boundaries land before the transfer completes, slow enough per chunk From de8d41fab690d0d7993a3f79a38b462b6ef5a664 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 19:27:31 +0000 Subject: [PATCH 06/20] test(comfyui): pin that an unusable setup runtime fails the default selection The previous commit makes the ComfyUI install recover the runtime `rocm setup` created before counting ready runtimes, as the engine paths do. A recovery that fails (the setup folder's runtime no longer passes the activation check) is an error there too; pin it for ComfyUI, with the registry left untouched. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index fc6b2689f4..647296f75a 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -1130,6 +1130,37 @@ mod tests { Ok(()) } + /// Recovering the setup runtime can fail, and the ComfyUI path now reports + /// that failure as the engine paths do, rather than carrying on as if setup + /// had named nothing: a setup folder whose runtime is no longer usable is + /// named, with the reason, and nothing is written to the registry. + #[test] + fn comfyui_selection_reports_a_setup_runtime_that_is_no_longer_usable() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-unusable"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let message = select_runtime(&paths, &config, None) + .expect_err("an unusable setup runtime is an error") + .to_string(); + + assert!( + message.contains("is not usable") && message.contains(&manifest.runtime_key), + "got: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + /// The two paths that pick "the one ready runtime" must not drift apart: /// for each registry state below, ComfyUI selects a runtime exactly when /// the engine paths do, and the same one. Both reach the rule through From 8fdc0fc7999936d91d26ce9d31685449826279cd Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 18:47:33 +0000 Subject: [PATCH 07/20] refactor(runtime): one definition of a non-empty install root has_nontrivial_directory_contents (therock.rs) and runtime_install_root_has_payload (runtime_registry.rs) were two copies of the same rule: an install root has a payload when it holds an entry whose name does not start with a dot, so the runtime's own .rocm-cli-runtime.json marker does not count. They answered every input identically except a path that does not exist (or a dangling symlink root): the first said "empty", the second failed with "failed to read". No caller can reach that row. Activation validation rejects a root that is not a directory ("install root is missing") before it asks, tarball extraction asks right after create_dir_all, and the update check maps any failure to "not installed" (unwrap_or(false)) anyway. Keep the first one's answer (a root that is not there holds nothing, so asking never needs an error path for it) and move it to rocm-core as install_root_has_payload next to the other install-root helpers; a root that exists but cannot be listed is still an error. Tarball extraction, the update path's check that a repair's replacement is installed, and activation validation of a tarball runtime now all ask it. The extraction refusal moves into ensure_tarball_install_target_is_clean so the refusal can be tested with its message and the files it leaves alone. The two per-copy tables from the previous commit are replaced by one table over the shared function, plus a test per caller that goes through it: a marker-only root has no payload for validation (message and unusable status asserted together), for the extraction guard, and for the replacement check. Re-introducing a private copy in a caller, or changing the shared predicate, turns these red. No user-visible change. Signed-off-by: Jussi Elo --- apps/rocm/src/runtime_registry.rs | 153 +++++++++------------------- apps/rocm/src/therock.rs | 156 ++++++++--------------------- apps/rocm/src/therock/update.rs | 60 ++++++++++- crates/rocm-core/src/lib.rs | 20 ++-- crates/rocm-core/src/runtime.rs | 161 ++++++++++++++++++++++++++++++ docs/architecture.md | 8 +- 6 files changed, 321 insertions(+), 237 deletions(-) diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 2f94a34448..f39447d4e7 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -628,7 +628,7 @@ fn validate_wheel_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Resul } fn validate_tarball_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Result<()> { - if runtime_install_root_has_payload(&manifest.install_root)? { + if rocm_core::install_root_has_payload(&manifest.install_root)? { Ok(()) } else { bail!( @@ -638,16 +638,6 @@ fn validate_tarball_runtime_manifest(manifest: &InstalledRuntimeManifest) -> Res } } -fn runtime_install_root_has_payload(path: &Path) -> Result { - for entry in fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))? { - let entry = entry?; - if !entry.file_name().to_string_lossy().starts_with('.') { - return Ok(true); - } - } - Ok(false) -} - pub(crate) fn paths_equivalent(left: &Path, right: &Path) -> bool { let left = normalize_path_for_compare(left); let right = normalize_path_for_compare(right); @@ -947,103 +937,56 @@ mod tests { Ok(()) } - /// What `runtime_install_root_has_payload` answers for each kind of - /// install-root path: the same table `therock.rs` holds - /// `has_nontrivial_directory_contents` to, which differs in exactly the two - /// rows where the path does not exist. + /// Activation validation of a tarball runtime follows + /// `rocm_core::install_root_has_payload`: the root's own `.rocm-cli-runtime.json` + /// marker is a dot-entry and does not make a payload, so a root holding only + /// that is reported unusable (by the message the runtime listing prints), and + /// one more entry makes it ready. A root that is gone is reported by the + /// earlier "install root is missing" check, never reaching the payload rule. #[test] - fn install_root_payload_characterisation() { - let base = tempfile::tempdir().expect("scratch dir"); - let mut seen = Vec::new(); - for (label, path) in install_root_cases(base.path()) { - let outcome = match runtime_install_root_has_payload(&path) { - Ok(true) => "payload", - Ok(false) => "empty", - Err(_) => "error", - }; - seen.push((label, outcome)); - } - let mut expected = vec![ - ("missing path", "error"), - ("empty dir", "empty"), - ("dotfile only", "empty"), - ("dot directory only", "empty"), - ("regular file", "payload"), - ("subdirectory", "payload"), - ("root is a regular file", "error"), - ]; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - expected.push(("dangling symlink entry", "payload")); - expected.push(("root is a symlink to a populated dir", "payload")); - expected.push(("root is a dangling symlink", "error")); - // Absent from `seen` when running as root, which reads a mode-000 - // directory anyway; then the case is neither built nor asserted. - if seen.iter().any(|(label, _)| *label == "unreadable dir") { - expected.push(("unreadable dir", "error")); - } - fs::set_permissions( - base.path().join("locked"), - fs::Permissions::from_mode(0o755), + fn tarball_validation_follows_the_shared_payload_rule() { + let (root, _) = test_paths("tarball-shared-payload-rule"); + let install_root = root.join("tarball-runtime"); + fs::create_dir_all(&install_root).unwrap(); + fs::write(install_root.join(".rocm-cli-runtime.json"), b"{}").unwrap(); + let manifest = InstalledRuntimeManifest { + format: "tarball".to_owned(), + install_root: install_root.clone(), + ..test_runtime_manifest( + "release-tarball-gfx120X-all-7-13-0", + "therock-release:gfx120X-all", + 1, ) - .ok(); - } - assert_eq!(sorted(seen), sorted(expected)); - } + }; - fn sorted(mut rows: Vec<(&'static str, &'static str)>) -> Vec<(&'static str, &'static str)> { - rows.sort_unstable(); - rows - } + let error = validate_runtime_manifest_for_activation(&manifest) + .expect_err("a root with only the dot-marker has no payload") + .to_string(); + assert_eq!( + error, + format!( + "tarball runtime install root has no payload files: {}", + install_root.display() + ) + ); + assert_eq!( + runtime_usability_status(&manifest), + format!("unusable ({error})") + ); - /// One path per input kind. An unreadable directory is only built when the - /// process cannot read it anyway (a root user can, and then the case says - /// nothing). - fn install_root_cases(base: &Path) -> Vec<(&'static str, PathBuf)> { - let mut cases = vec![("missing path", base.join("missing"))]; - let dir = |name: &str| { - let path = base.join(name); - fs::create_dir_all(&path).expect("create case dir"); - path - }; - cases.push(("empty dir", dir("empty"))); - let dotfile = dir("dotfile"); - fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); - cases.push(("dotfile only", dotfile)); - let dotdir = dir("dotdir"); - fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); - cases.push(("dot directory only", dotdir)); - let file = dir("file"); - fs::write(file.join("payload.bin"), b"x").expect("write payload"); - cases.push(("regular file", file)); - let sub = dir("sub"); - fs::create_dir_all(sub.join("bin")).expect("create subdir"); - cases.push(("subdirectory", sub)); - let not_a_dir = base.join("not-a-dir"); - fs::write(¬_a_dir, b"x").expect("write file root"); - cases.push(("root is a regular file", not_a_dir)); - #[cfg(unix)] - { - use std::os::unix::fs::{PermissionsExt, symlink}; - let dangling = dir("dangling-entry"); - symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); - cases.push(("dangling symlink entry", dangling)); - let link_root = base.join("link-root"); - symlink(base.join("sub"), &link_root).expect("symlink root"); - cases.push(("root is a symlink to a populated dir", link_root)); - let dangling_root = base.join("dangling-root"); - symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); - cases.push(("root is a dangling symlink", dangling_root)); - let locked = dir("locked"); - fs::write(locked.join("payload.bin"), b"x").expect("write payload"); - fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); - if fs::read_dir(&locked).is_err() { - cases.push(("unreadable dir", locked)); - } else { - fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); - } - } - cases + fs::write(install_root.join("rocm-install-file"), b"x").unwrap(); + validate_runtime_manifest_for_activation(&manifest) + .expect("one entry beside the marker is a payload"); + assert_eq!(runtime_usability_status(&manifest), "ready"); + + fs::remove_dir_all(&install_root).unwrap(); + let error = validate_runtime_manifest_for_activation(&manifest) + .expect_err("a missing root is unusable") + .to_string(); + assert_eq!( + error, + format!("install root is missing: {}", install_root.display()) + ); + fs::remove_dir_all(root).ok(); } } diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 2939fbd6d4..f20fdfdcc6 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -960,12 +960,7 @@ fn install_tarball_runtime( fs::create_dir_all(paths.cache_dir.join("therock"))?; fs::create_dir_all(&install_root)?; - if has_nontrivial_directory_contents(&install_root)? { - bail!( - "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", - install_root.display() - ); - } + ensure_tarball_install_target_is_clean(&install_root)?; if let Some(warning) = preflight_tarball_space(&artifact.url, &cache_path, &install_root)? { let _ = writeln!(output, " {warning}"); @@ -1114,21 +1109,18 @@ fn resolved_install_root( rocm_core::resolve_path_through_symlinks(&requested) } -fn has_nontrivial_directory_contents(path: &Path) -> Result { - if !path.exists() { - return Ok(false); - } - let entries = - fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))?; - for entry in entries { - let entry = entry?; - let name = entry.file_name(); - if name.to_string_lossy().starts_with('.') { - continue; - } - return Ok(true); +/// Refuses to extract a tarball into a root that already holds a payload. +/// +/// "Holds a payload" is [`rocm_core::install_root_has_payload`]: dot-entries (the +/// runtime marker) do not count, so a root that has only been prepared passes. +fn ensure_tarball_install_target_is_clean(install_root: &Path) -> Result<()> { + if rocm_core::install_root_has_payload(install_root)? { + bail!( + "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", + install_root.display() + ); } - Ok(false) + Ok(()) } fn venv_python_path(install_root: &Path) -> PathBuf { @@ -1709,102 +1701,40 @@ mod tests { ); } - /// What `has_nontrivial_directory_contents` answers for each kind of - /// install-root path, so the sibling rule in `runtime_registry.rs` can be held - /// to the same table. + /// The extraction guard refuses a root that holds a payload and says so, and + /// it follows `rocm_core::install_root_has_payload` for what that means: a + /// root that does not exist yet (the case `install_tarball_runtime` hits when + /// `--prefix` names a new folder) and one holding only dot-entries pass. + /// Refusing must leave the existing files where they were. #[test] - fn nontrivial_directory_contents_characterisation() { + fn the_tarball_install_guard_follows_the_shared_payload_rule() { let base = tempfile::tempdir().expect("scratch dir"); - let mut seen = Vec::new(); - for (label, path) in install_root_cases(base.path()) { - let outcome = match has_nontrivial_directory_contents(&path) { - Ok(true) => "payload", - Ok(false) => "empty", - Err(_) => "error", - }; - seen.push((label, outcome)); - } - let mut expected = vec![ - ("missing path", "empty"), - ("empty dir", "empty"), - ("dotfile only", "empty"), - ("dot directory only", "empty"), - ("regular file", "payload"), - ("subdirectory", "payload"), - ("root is a regular file", "error"), - ]; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - expected.push(("dangling symlink entry", "payload")); - expected.push(("root is a symlink to a populated dir", "payload")); - expected.push(("root is a dangling symlink", "empty")); - // Absent from `seen` when running as root, which reads a mode-000 - // directory anyway; then the case is neither built nor asserted. - if seen.iter().any(|(label, _)| *label == "unreadable dir") { - expected.push(("unreadable dir", "error")); - } - fs::set_permissions( - base.path().join("locked"), - fs::Permissions::from_mode(0o755), - ) - .ok(); - } - assert_eq!(sorted(seen), sorted(expected)); - } - fn sorted(mut rows: Vec<(&'static str, &'static str)>) -> Vec<(&'static str, &'static str)> { - rows.sort_unstable(); - rows - } + ensure_tarball_install_target_is_clean(&base.path().join("not-created-yet")) + .expect("a missing root is clean"); - /// One path per input kind. An unreadable directory is only built when the - /// process cannot read it anyway (a root user can, and then the case says - /// nothing). - fn install_root_cases(base: &Path) -> Vec<(&'static str, PathBuf)> { - let mut cases = vec![("missing path", base.join("missing"))]; - let dir = |name: &str| { - let path = base.join(name); - fs::create_dir_all(&path).expect("create case dir"); - path - }; - cases.push(("empty dir", dir("empty"))); - let dotfile = dir("dotfile"); - fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); - cases.push(("dotfile only", dotfile)); - let dotdir = dir("dotdir"); - fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); - cases.push(("dot directory only", dotdir)); - let file = dir("file"); - fs::write(file.join("payload.bin"), b"x").expect("write payload"); - cases.push(("regular file", file)); - let sub = dir("sub"); - fs::create_dir_all(sub.join("bin")).expect("create subdir"); - cases.push(("subdirectory", sub)); - let not_a_dir = base.join("not-a-dir"); - fs::write(¬_a_dir, b"x").expect("write file root"); - cases.push(("root is a regular file", not_a_dir)); - #[cfg(unix)] - { - use std::os::unix::fs::{PermissionsExt, symlink}; - let dangling = dir("dangling-entry"); - symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); - cases.push(("dangling symlink entry", dangling)); - let link_root = base.join("link-root"); - symlink(base.join("sub"), &link_root).expect("symlink root"); - cases.push(("root is a symlink to a populated dir", link_root)); - let dangling_root = base.join("dangling-root"); - symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); - cases.push(("root is a dangling symlink", dangling_root)); - let locked = dir("locked"); - fs::write(locked.join("payload.bin"), b"x").expect("write payload"); - fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); - if fs::read_dir(&locked).is_err() { - cases.push(("unreadable dir", locked)); - } else { - fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); - } - } - cases + let marker_only = base.path().join("marker-only"); + fs::create_dir_all(&marker_only).unwrap(); + fs::write(marker_only.join(".rocm-cli-runtime.json"), b"{}").unwrap(); + ensure_tarball_install_target_is_clean(&marker_only) + .expect("a root with only the dot-marker is clean"); + + let populated = base.path().join("populated"); + fs::create_dir_all(populated.join("bin")).unwrap(); + fs::write(populated.join("bin").join("tool"), b"x").unwrap(); + let error = ensure_tarball_install_target_is_clean(&populated) + .expect_err("a root with a payload is refused") + .to_string(); + assert_eq!( + error, + format!( + "tarball install target {} is not empty; choose a clean prefix or remove the old extraction first", + populated.display() + ) + ); + assert!( + populated.join("bin").join("tool").is_file(), + "the refusal must not touch what is already there" + ); } } diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 22b4e3c96a..9d399d128d 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -17,9 +17,10 @@ //! //! Builds on `family`, `io`, `process`, `python`, `resolve`, `source` and //! `version`, and on `runtime_registry` for the manifests; `runtime_key`, -//! `wheel_runtime_key` and `has_nontrivial_directory_contents`, which name and -//! inspect the install a plan targets, stay in `therock.rs` with the other keys -//! and roots helpers. +//! `wheel_runtime_key`, which name the install a plan targets, stay in +//! `therock.rs` with the other keys and roots helpers. Whether an install root +//! holds anything is `rocm_core::install_root_has_payload`, the one rule the +//! install and activation paths share. use super::family::family_override_or_recovered_arch; use super::io::write_file_atomically; @@ -34,7 +35,7 @@ use super::resolve::{ }; use super::source::{SourceLayout, TheRockChannel}; use super::version::{runtime_version_display, version_relation}; -use super::{has_nontrivial_directory_contents, runtime_key, wheel_runtime_key}; +use super::{runtime_key, wheel_runtime_key}; use crate::runtime_registry::{ InstalledRuntimeManifest, WheelRuntimeComposition, load_runtime_manifests, wheel_composition_device_target, @@ -295,7 +296,7 @@ fn replacement_runtime_is_installed( && candidate.format == source.format && candidate.family == source.family && candidate.wheel_composition.as_ref() == Some(required_composition) - && has_nontrivial_directory_contents(&candidate.install_root).unwrap_or(false) + && rocm_core::install_root_has_payload(&candidate.install_root).unwrap_or(false) }) } @@ -1071,6 +1072,55 @@ mod tests { fs::remove_dir_all(root).ok(); } + /// A replacement counts as installed by `rocm_core::install_root_has_payload`: + /// a root holding only the runtime's own dot-marker was prepared but never + /// filled, so the repair is still on offer; one entry beside the marker is a + /// real install, so it is not. The runtime directory vanishing is covered by + /// `an_installed_replacement_sibling_suppresses_repeat_repair`. + #[test] + fn a_replacement_counts_as_installed_by_the_shared_payload_rule() { + let required = test_wheel_composition("gfx942"); + let target_runtime_key = wheel_runtime_key(TheRockChannel::Release, "7.14.0", &required); + let mut source = test_runtime_manifest( + "release-wheel-multi-arch-7-14-0", + "therock-release:gfx94X-dcgpu", + 1, + ); + source.version = "7.14.0".to_owned(); + let mut replacement = + test_runtime_manifest(&target_runtime_key, "therock-release:gfx94X-dcgpu", 2); + replacement.version = "7.14.0".to_owned(); + replacement.wheel_composition = Some(required.clone()); + let (root, _) = test_paths("replacement-shared-payload-rule"); + replacement.install_root = root.join("replacement"); + fs::create_dir_all(&replacement.install_root).unwrap(); + fs::write( + replacement.install_root.join(".rocm-cli-runtime.json"), + b"{}", + ) + .unwrap(); + let manifests = vec![source.clone(), replacement.clone()]; + let freshness = |manifests: &[InstalledRuntimeManifest]| { + runtime_freshness_with_manifests( + manifests, + &source, + "7.14.0", + Some(&required), + &target_runtime_key, + ) + }; + + assert_eq!( + freshness(&manifests), + RuntimeFreshness::RepairAvailable, + "a root with only the dot-marker has no payload, so it is not a replacement" + ); + + fs::write(replacement.install_root.join("installed.marker"), b"ok").unwrap(); + assert_eq!(freshness(&manifests), RuntimeFreshness::UpToDate); + fs::remove_dir_all(root).ok(); + } + #[test] fn update_json_reports_no_managed_runtimes_as_empty_list() -> Result<()> { let (root, paths) = test_paths("update-json-empty"); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 69e2da9599..a1052133e5 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -193,16 +193,16 @@ use runtime::env_path_override; pub use runtime::{ RUNTIME_LIBRARY_PATH_ENV, RuntimeHost, RuntimePlatform, current_executable_path, default_cache_dir, default_config_dir, default_data_dir, default_interactive_shell_program, - managed_logs_dir, managed_pip_cache_dir, managed_runtime_cache_dir, managed_runtime_data_root, - managed_tools_dir, managed_uv_cache_dir, normalize_runtime_path_for_host, - normalize_runtime_path_for_storage, normalize_runtime_path_text_for_host, - normalize_runtime_path_text_for_platform, normalize_runtime_path_text_for_storage, - platform_binary_name, prepend_runtime_path, resolve_path_through_symlinks, - runtime_directory_label, runtime_drive_root_for_key, runtime_drive_roots, runtime_exe_suffix, - runtime_home_dir, runtime_install_root_is_protected, runtime_is_linux, runtime_is_windows, - runtime_os_name, runtime_path_for_child, runtime_path_for_windows_child, - runtime_path_is_same_or_inside, runtime_path_list_join, runtime_path_list_split, - runtime_path_sort_key, runtime_path_text_is_absolute_for_host, + install_root_has_payload, managed_logs_dir, managed_pip_cache_dir, managed_runtime_cache_dir, + managed_runtime_data_root, managed_tools_dir, managed_uv_cache_dir, + normalize_runtime_path_for_host, normalize_runtime_path_for_storage, + normalize_runtime_path_text_for_host, normalize_runtime_path_text_for_platform, + normalize_runtime_path_text_for_storage, platform_binary_name, prepend_runtime_path, + resolve_path_through_symlinks, runtime_directory_label, runtime_drive_root_for_key, + runtime_drive_roots, runtime_exe_suffix, runtime_home_dir, runtime_install_root_is_protected, + runtime_is_linux, runtime_is_windows, runtime_os_name, runtime_path_for_child, + runtime_path_for_windows_child, runtime_path_is_same_or_inside, runtime_path_list_join, + runtime_path_list_split, runtime_path_sort_key, runtime_path_text_is_absolute_for_host, runtime_path_text_is_absolute_for_platform, runtime_paths_equivalent, runtime_python_activation_hint, runtime_python_activation_script, runtime_python_bin_dir_name, runtime_python_env_bin_dir, runtime_python_executable_in_env, runtime_python_executable_name, diff --git a/crates/rocm-core/src/runtime.rs b/crates/rocm-core/src/runtime.rs index 3535035b07..17a1dfda30 100644 --- a/crates/rocm-core/src/runtime.rs +++ b/crates/rocm-core/src/runtime.rs @@ -695,6 +695,35 @@ pub fn runtime_install_root_is_protected(path: &Path) -> bool { .any(|root| runtime_path_is_same_or_inside(&path, root)) } +/// Whether an install root holds anything the install put there. +/// +/// Dot-entries do not count: the runtime's own `.rocm-cli-runtime.json` marker is +/// one, and a root that holds only that has no payload. Any other entry does, +/// whatever it is: a file, a directory, or a symlink that points nowhere. A path +/// that does not exist holds nothing, so it answers `false` rather than failing; +/// that is what lets a fresh install ask about a root it has not created yet. A +/// root that exists but cannot be listed (not a directory, no permission) is an +/// error, never a quiet `false`. +/// +/// This is the one rule for "is this install root empty" in the workspace: the +/// tarball extraction refuses a target that has a payload, a repair counts a +/// sibling as installed only when it has one, and activation validation rejects a +/// tarball runtime that has none. +pub fn install_root_has_payload(path: &Path) -> Result { + if !path.exists() { + return Ok(false); + } + let entries = + std::fs::read_dir(path).with_context(|| format!("failed to read {}", path.display()))?; + for entry in entries { + let entry = entry?; + if !entry.file_name().to_string_lossy().starts_with('.') { + return Ok(true); + } + } + Ok(false) +} + #[cfg_attr(not(test), allow(dead_code))] #[derive(Debug, Clone, Copy, Eq, PartialEq)] enum RuntimePathSeparator { @@ -1819,3 +1848,135 @@ mod tests { } } } + +#[cfg(test)] +mod install_root_payload_tests { + use super::*; + use std::fs; + + /// One path per kind of install root. An unreadable directory is only built + /// when this process cannot read it anyway (a root user can, and then the case + /// says nothing). + fn cases(base: &Path) -> Vec<(&'static str, PathBuf)> { + let mut cases = vec![("missing path", base.join("missing"))]; + let dir = |name: &str| { + let path = base.join(name); + fs::create_dir_all(&path).expect("create case dir"); + path + }; + cases.push(("empty dir", dir("empty"))); + let dotfile = dir("dotfile"); + fs::write(dotfile.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + cases.push(("dotfile only", dotfile)); + let dotdir = dir("dotdir"); + fs::create_dir_all(dotdir.join(".hidden")).expect("create dot dir"); + cases.push(("dot directory only", dotdir)); + let file = dir("file"); + fs::write(file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("regular file", file)); + let sub = dir("sub"); + fs::create_dir_all(sub.join("bin")).expect("create subdir"); + cases.push(("subdirectory", sub)); + let marker_and_file = dir("marker-and-file"); + fs::write(marker_and_file.join(".rocm-cli-runtime.json"), b"{}").expect("write marker"); + fs::write(marker_and_file.join("payload.bin"), b"x").expect("write payload"); + cases.push(("marker plus a regular file", marker_and_file)); + let not_a_dir = base.join("not-a-dir"); + fs::write(¬_a_dir, b"x").expect("write file root"); + cases.push(("root is a regular file", not_a_dir)); + #[cfg(unix)] + { + use std::os::unix::fs::{PermissionsExt, symlink}; + let dangling = dir("dangling-entry"); + symlink(base.join("nowhere"), dangling.join("link")).expect("dangling entry"); + cases.push(("dangling symlink entry", dangling)); + let link_root = base.join("link-root"); + symlink(base.join("sub"), &link_root).expect("symlink root"); + cases.push(("root is a symlink to a populated dir", link_root)); + let dangling_root = base.join("dangling-root"); + symlink(base.join("nowhere"), &dangling_root).expect("dangling root"); + cases.push(("root is a dangling symlink", dangling_root)); + let locked = dir("locked"); + fs::write(locked.join("payload.bin"), b"x").expect("write payload"); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).expect("lock dir"); + if fs::read_dir(&locked).is_err() { + cases.push(("unreadable dir", locked)); + } else { + fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok(); + } + } + cases + } + + /// The single definition of a non-empty install root, over every input kind. + /// Each caller in `apps/rocm` goes through this function (see + /// `tarball_validation_follows_the_shared_payload_rule`, + /// `the_tarball_install_guard_follows_the_shared_payload_rule` and + /// `a_replacement_counts_as_installed_by_the_shared_payload_rule` there), so a + /// row changing here changes all three together. + #[test] + fn install_root_has_payload_answers_each_input_kind() { + let base = tempfile::tempdir().expect("scratch dir"); + let mut seen: Vec<(&str, String)> = cases(base.path()) + .into_iter() + .map(|(label, path)| { + let outcome = match install_root_has_payload(&path) { + Ok(true) => "payload", + Ok(false) => "empty", + Err(_) => "error", + }; + (label, outcome.to_owned()) + }) + .collect(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + base.path().join("locked"), + fs::Permissions::from_mode(0o755), + ) + .ok(); + } + let mut expected: Vec<(&str, String)> = vec![ + ("missing path", "empty"), + ("empty dir", "empty"), + ("dotfile only", "empty"), + ("dot directory only", "empty"), + ("regular file", "payload"), + ("subdirectory", "payload"), + ("marker plus a regular file", "payload"), + ("root is a regular file", "error"), + ] + .into_iter() + .map(|(label, outcome)| (label, outcome.to_owned())) + .collect(); + #[cfg(unix)] + { + expected.extend([ + ("dangling symlink entry", "payload".to_owned()), + ("root is a symlink to a populated dir", "payload".to_owned()), + ("root is a dangling symlink", "empty".to_owned()), + ]); + // Absent from `seen` when running as root, which reads a mode-000 + // directory anyway; then the case is neither built nor asserted. + if seen.iter().any(|(label, _)| *label == "unreadable dir") { + expected.push(("unreadable dir", "error".to_owned())); + } + } + seen.sort(); + expected.sort(); + assert_eq!(seen, expected); + } + + #[test] + fn a_root_that_cannot_be_listed_names_the_path_in_its_error() { + let base = tempfile::tempdir().expect("scratch dir"); + let file = base.path().join("not-a-dir"); + fs::write(&file, b"x").expect("write file root"); + let error = install_root_has_payload(&file).expect_err("a file cannot be listed"); + assert!( + format!("{error:#}").contains(&format!("failed to read {}", file.display())), + "{error:#}" + ); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index df8fc9161e..0c789a98e5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,7 +71,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. -- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. +- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it; the tarball one asks `rocm_core::install_root_has_payload` whether the root holds a payload). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. @@ -87,7 +87,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and `real_os_release`, the real host's os-release that every one of these plans is built from. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions and `real_os_release` are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), and [`driver_install.rs`](../apps/rocm/src/driver_install.rs) imports `real_os_release` from here directly. `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `real_os_release` ignores a simulated host root. -- [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token`, `slugify` and `has_nontrivial_directory_contents`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read six of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key`, `wheel_runtime_key` and `has_nontrivial_directory_contents` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) stays in [`main.rs`](../apps/rocm/src/main.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). +- [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) stays in [`main.rs`](../apps/rocm/src/main.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. - [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. - [`therock/process.rs`](../apps/rocm/src/therock/process.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for scratch directories (split out of `therock.rs`): running and capturing child commands (`run_command*`, `capture_command_output*`, `capture_python_stdout*`, `run_uv_progress_command`, and the Windows temp-file variants with `windows_child_path`), the bounded wait (`wait_with_output_bounded`), the progress-line output with its `--json` suppression guard (`SuppressProgressOutput`, `progress_line`), and unpacking the SDK tarball (`extract_tarball`, `extract_tarball_and_discard_archive`, which is why it sits here and not in `io.rs`: it spawns `tar` through `run_command`). Its `archive_properties` property tests run on Unix only. The process-wide environment lock these tests take is the single `PROCESS_ENV_TEST_LOCK` in [`test_support.rs`](../apps/rocm/src/test_support.rs), shared with the tests left in `therock.rs`. @@ -100,7 +100,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. - [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. -- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key`, `wheel_runtime_key` and `has_nontrivial_directory_contents`, which name and inspect the install a plan targets, stay in `therock.rs`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. +- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. - [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK`. - [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_chat_rocm_command_safety`, which refuses a public-bind, non-loopback or CPU `serve` and a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_chat_rocm_command_safety` here instead of through `chat`. `chat.rs` calls into it and re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). - [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. The two modules depend on each other: `uninstall_plan.rs` reads this module's `pub(crate)` `PathEdit`, `PATH_EDIT_RECORD`, `plan_path_edits`, `plan_is_empty`, `removal_count` and `render_extra_plan`. @@ -216,7 +216,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. -- [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default config/data/cache directories, and path-text and path-list helpers (no module doc comment). +- [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names, default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). - [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose/engine.rs`). - [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. - [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. From b4d94d3bf4b14efb7bd9c4c9070d26bae3e092b0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 19:46:20 +0000 Subject: [PATCH 08/20] test(system_packages): pin that the install hints read the real host's os-release The OpenMPI, libatomic and libnuma install hints name package-manager commands for the machine the process runs on, so a simulated host root must not choose their distro. rocm-core's private real_os_release (openmpi.rs) and the apps/rocm one (system_packages.rs) are the same one-line call, read_os_release with the identity resolver; the apps copy already has a test for it, the rocm-core copy had none. Add one that plants a root whose os-release names another distro and checks all three hints are unchanged by it. It lives with the apps copy's test, in the rocm binary's test process, so the environment variable it sets cannot overlap the rocm-core tests that read host paths. Swapping the identity resolver in openmpi.rs for host_path_for_str turns it red. Tests only. Signed-off-by: Jussi Elo --- apps/rocm/src/system_packages.rs | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/apps/rocm/src/system_packages.rs b/apps/rocm/src/system_packages.rs index 4e3b5b78b6..9d2807970c 100644 --- a/apps/rocm/src/system_packages.rs +++ b/apps/rocm/src/system_packages.rs @@ -605,4 +605,52 @@ mod tests { ), } } + + /// The OpenMPI, libatomic and libnuma install hints (`rocm_core::openmpi`) + /// name package-manager commands for the machine the process runs on, like + /// the plans in this module, so a simulated host root must not choose their + /// distro either: with a root whose `/etc/os-release` names another + /// distribution, each hint is still the one the real host's file produces. + /// A hint that read through `host_path` would be the planted distro's (the + /// manual advice, as `planted-distro` maps to no package manager). + #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] + #[test] + fn the_install_hints_read_the_real_host_and_ignore_a_simulated_root() { + let root = std::env::temp_dir().join(format!( + "rocm-os-release-hint-root-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()) + )); + fs::create_dir_all(root.join("etc")).expect("create planted /etc"); + fs::write(root.join("etc/os-release"), "ID=planted-distro\n") + .expect("write planted os-release"); + + let hints = || { + [ + rocm_core::openmpi::install_hint(), + rocm_core::openmpi::libatomic_install_hint(), + rocm_core::openmpi::libnuma_install_hint(), + ] + }; + let (real, simulated) = { + let mut env = ScopedTestEnv::new(); + let real = hints(); + env.set( + rocm_core::hardware_root::TEST_HOST_ROOT_ENV, + &root.to_string_lossy(), + ); + (real, hints()) + }; + fs::remove_dir_all(&root).ok(); + + assert_eq!(simulated, real); + if !real[0].plan_installs { + // A host that maps to no package manager reads the same as the + // planted file, so the assertion above held without telling which + // one was read. + eprintln!("host os-release names no known distribution; nothing proven"); + } + } } From e36e69d0524c353be80d4e8f488fb33b4858b2fb Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 19:55:40 +0000 Subject: [PATCH 09/20] refactor(hardware_root): one reader for the real host's os-release The OpenMPI, libatomic and libnuma install hints (rocm-core/openmpi.rs), the system-package plans and the driver plan (apps/rocm via system_packages.rs) each need the os-release of the machine the process runs on, not a simulated host root. openmpi.rs and system_packages.rs each carried their own real_os_release, identical line for line: read_os_release with the identity resolver. The e2e driver step spelled the same call a third time. They read the same two files in the same order with the same fallback and error rules, so nothing a user can see differs; the risk was one copy being edited and the others not. Move the reader to rocm-core as hardware_root::real_os_release, delete both private copies, and make the identity resolver private so a caller outside hardware_root.rs has no shared way to build a second reader. The apps copy's test (real_os_release_ignores_a_simulated_host_root) stays where it was and now calls the shared function; the hint test from the previous commit holds the three hints to the same rule, and a scan test in the rocm binary fails if any of its source files names read_os_release or the resolver. The shared function is no longer cfg(linux)-gated as openmpi's was; off Linux it reads the same paths and finds nothing, as the apps copy always did. Doc comments, docs/architecture.md and docs/testing.md that named the two copies are updated. Signed-off-by: Jussi Elo --- apps/rocm/src/driver_install.rs | 3 +- apps/rocm/src/driver_plan.rs | 6 +- apps/rocm/src/system_packages.rs | 84 +++++++++++++++----- crates/rocm-core/src/hardware_root.rs | 34 +++++--- crates/rocm-core/src/openmpi.rs | 24 ++---- docs/architecture.md | 6 +- docs/testing.md | 17 +++- tests/e2e-cucumber/features/examine.feature | 6 +- tests/e2e-cucumber/tests/e2e/driver_steps.rs | 7 +- 9 files changed, 120 insertions(+), 67 deletions(-) diff --git a/apps/rocm/src/driver_install.rs b/apps/rocm/src/driver_install.rs index d2f4a281c8..dad8008274 100644 --- a/apps/rocm/src/driver_install.rs +++ b/apps/rocm/src/driver_install.rs @@ -37,7 +37,6 @@ use crate::driver_plan::{ DriverCommandPhase, DriverInstallPlan, PrivilegeEscalation, build_driver_install_plan_read, render_driver_install_plan, }; -use crate::system_packages::real_os_release; /// Whether `rocm install driver` goes on to run the plan's commands. Pure so /// the gate that keeps native Windows from installing anything (its plan is @@ -62,7 +61,7 @@ pub(crate) fn install_driver( ) -> std::result::Result { let examine = ExamineSummary::gather().map_err(|source| DriverInstallError::new(source, false))?; - let os_release = real_os_release(); + let os_release = rocm_core::hardware_root::real_os_release(); // The only place the real privilege level is read; every builder below takes // it as a parameter so both branches stay testable on any host. let plan = diff --git a/apps/rocm/src/driver_plan.rs b/apps/rocm/src/driver_plan.rs index 9b8291d017..74e3eef843 100644 --- a/apps/rocm/src/driver_plan.rs +++ b/apps/rocm/src/driver_plan.rs @@ -1443,9 +1443,9 @@ pub(crate) mod tests { } /// `build_driver_install_plan_read` is the one seam `install_driver` hands - /// its `real_os_release()` result to. Both are now one-line delegations - /// to `hardware_root::read_os_release`, so reverting either back to a raw - /// `/etc/os-release` read is a type error here, not a silent regression. + /// `hardware_root::real_os_release()`'s result to, so reverting that back to + /// a raw `/etc/os-release` read is a type error here, not a silent + /// regression. /// This still pins the behaviour at the seam itself: an `OsRelease` that /// says the fallback path was read plans exactly what a primary-path read /// with the same content would. diff --git a/apps/rocm/src/system_packages.rs b/apps/rocm/src/system_packages.rs index 9d2807970c..22d7dcde32 100644 --- a/apps/rocm/src/system_packages.rs +++ b/apps/rocm/src/system_packages.rs @@ -13,8 +13,9 @@ //! `rocm install sdk` (`SdkInstallConsents`) stays in `runtime_install_finalize.rs`, //! which passes the plain `system_packages` bool in. That module re-exports //! `ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch` and -//! `ensure_libnuma_for_torch` for `install_cmd.rs` and `engines_cmd.rs`, and -//! `driver_install.rs` reads `real_os_release` from here directly. +//! `ensure_libnuma_for_torch` for `install_cmd.rs` and `engines_cmd.rs`. Every plan +//! here reads the real host's os-release through +//! `rocm_core::hardware_root::real_os_release`. use std::process::{Command as ProcessCommand, Stdio}; @@ -62,18 +63,6 @@ const fn system_package_install_action( } } -/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` -/// when the former is absent (per `os-release(5)`). -/// -/// Deliberately not routed through `rocm_core::host_path`: every caller builds -/// a package-manager plan (`apt`, `dnf`, `zypper`) that then runs against the -/// real machine, so a simulated host root must not choose its distro. The descriptive distro name `examine` reports comes from -/// `ExamineSummary::gather`'s `distro` field, which does follow a simulated -/// root. -pub(crate) fn real_os_release() -> rocm_core::hardware_root::OsRelease { - rocm_core::hardware_root::read_os_release(rocm_core::hardware_root::identity_path_for_str) -} - /// Run a command given as an argv vector directly, without going through a shell. /// /// Used for [`run_system_package_install_plan`], whose commands are modeled as @@ -117,7 +106,7 @@ pub(crate) fn ensure_openmpi_for_vllm(approved: bool) -> Result<()> { } let plan = rocm_core::openmpi::plan_for_os_release_read( - &real_os_release(), + &rocm_core::hardware_root::real_os_release(), rocm_core::openmpi::build_openmpi_install_plan, ); @@ -291,7 +280,10 @@ fn ensure_torch_runtime_dep(approved: bool, dep: &TorchRuntimeDep) { return; } - let plan = rocm_core::openmpi::plan_for_os_release_read(&real_os_release(), dep.build_plan); + let plan = rocm_core::openmpi::plan_for_os_release_read( + &rocm_core::hardware_root::real_os_release(), + dep.build_plan, + ); println!("{} setup", dep.name); println!(" reason: {}", dep.reason); @@ -440,7 +432,6 @@ fn run_system_package_install_plan( #[cfg(test)] mod tests { - #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] use std::fs; use super::*; @@ -533,8 +524,10 @@ mod tests { /// The package-manager plans run against the real machine, so a simulated /// host root must not change the distro they are built for: under a root - /// whose `/etc/os-release` names a different distro, `real_os_release` - /// still returns the real file. + /// whose `/etc/os-release` names a different distro, + /// `rocm_core::hardware_root::real_os_release` still returns the real file. + /// Every caller (the driver, OpenMPI, libatomic and libnuma plans and hints) + /// reads through that one function. #[cfg(all(feature = "e2e-test-hooks", target_os = "linux"))] #[test] fn real_os_release_ignores_a_simulated_host_root() { @@ -558,7 +551,7 @@ mod tests { rocm_core::hardware_root::TEST_HOST_ROOT_ENV, &root.to_string_lossy(), ); - real_os_release() + rocm_core::hardware_root::real_os_release() }; fs::remove_dir_all(&root).ok(); @@ -569,8 +562,8 @@ mod tests { ); // Independent of `hardware_root::read_os_release`: rather than call // the exact same helper with the same identity resolver `real_os_release` - // wraps one line away (which would pass even if its `.context(...)` - // call or resolver closure broke), read both candidate paths directly + // wraps (which would pass even if the resolver it passes broke), read + // both candidate paths directly // and apply the same existence-only fallback rule inline. This mirrors // which variant is correct for each case, rather than accepting // `Absent` or `Unreadable` interchangeably — see `ExpectedOsRelease`'s @@ -606,6 +599,53 @@ mod tests { } } + /// Every package-manager plan this module and `driver_install.rs` build is + /// written for the real machine, so they all read its os-release through + /// `rocm_core::hardware_root::real_os_release`, whose own test + /// (`real_os_release_ignores_a_simulated_host_root`) proves it ignores a + /// simulated host root. A caller that read the file with `read_os_release` + /// and its own resolver would be a second reader that test does not cover, so + /// no source file of this binary may name `read_os_release` or the identity + /// resolver: the one place that does is `hardware_root.rs` in `rocm-core`. + #[test] + fn no_caller_in_this_binary_reads_the_real_os_release_by_hand() { + fn rust_files(dir: &std::path::Path, out: &mut Vec) { + for entry in fs::read_dir(dir).expect("a readable source directory") { + let path = entry.expect("a readable entry").path(); + if path.is_dir() { + rust_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") { + out.push(path); + } + } + } + + // Spelled in two halves so this file does not match its own scan. + let needles = [ + ["read_os_", "release("].concat(), + ["identity_path_", "for_str"].concat(), + ]; + let mut files = Vec::new(); + rust_files( + &std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut files, + ); + assert!(!files.is_empty(), "the scan found no source files"); + let offenders: Vec = files + .iter() + .filter(|path| { + let text = fs::read_to_string(path).expect("a readable source file"); + needles.iter().any(|needle| text.contains(needle.as_str())) + }) + .map(|path| path.display().to_string()) + .collect(); + assert!( + offenders.is_empty(), + "these files read os-release by hand instead of through \ + rocm_core::hardware_root::real_os_release: {offenders:?}" + ); + } + /// The OpenMPI, libatomic and libnuma install hints (`rocm_core::openmpi`) /// name package-manager commands for the machine the process runs on, like /// the plans in this module, so a simulated host root must not choose their diff --git a/crates/rocm-core/src/hardware_root.rs b/crates/rocm-core/src/hardware_root.rs index 86a5e3d9e2..c6298e9d45 100644 --- a/crates/rocm-core/src/hardware_root.rs +++ b/crates/rocm-core/src/hardware_root.rs @@ -90,20 +90,34 @@ pub fn host_path_for_str(path: &str) -> PathBuf { host_path(path) } -/// [`read_os_release`]'s `resolve` parameter for a caller that must always -/// read the real machine. +/// [`read_os_release`]'s `resolve` parameter for the real machine: the path +/// as written. A bare `PathBuf::from` closure is rejected the same way a bare +/// `host_path` is (see [`host_path_for_str`]), hence a named function. /// -/// Never routed through [`host_path`], because it builds a package-manager -/// plan or install hint that runs against the real host, so a simulated -/// host root must not choose its distro (see each caller's own doc comment -/// for why). A bare `PathBuf::from` is rejected the same way a bare -/// `host_path` is (see [`host_path_for_str`]), so every such caller passes -/// this instead of `|p| PathBuf::from(p)`. -#[must_use] -pub fn identity_path_for_str(path: &str) -> PathBuf { +/// Private: the only reader that must always see the real machine is +/// [`real_os_release`], and having one place that names this resolver is what +/// keeps a second, diverging copy of that reader from appearing elsewhere. +fn identity_path_for_str(path: &str) -> PathBuf { PathBuf::from(path) } +/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` +/// when the former is absent (per `os-release(5)`). +/// +/// The one reader for every caller that builds a package-manager plan or an +/// install hint (`apt`, `dnf`, `zypper`, ...): those commands run against the +/// real machine, so a simulated host root (see [`host_path`]) must not choose +/// the distro they are written for. Deliberately not routed through +/// [`host_path`] for that reason. The descriptive distro name and id that +/// `examine` reports come from [`read_os_release`] with [`host_path_for_str`] +/// instead, and do follow a simulated root. +/// +/// Reads the same files on every platform; off Linux they are simply absent. +#[must_use] +pub fn real_os_release() -> OsRelease { + read_os_release(identity_path_for_str) +} + #[cfg(feature = "e2e-test-hooks")] fn hardware_root() -> Option { std::env::var_os(TEST_HOST_ROOT_ENV) diff --git a/crates/rocm-core/src/openmpi.rs b/crates/rocm-core/src/openmpi.rs index 44b6a8843f..2f45df2da2 100644 --- a/crates/rocm-core/src/openmpi.rs +++ b/crates/rocm-core/src/openmpi.rs @@ -707,17 +707,6 @@ pub fn build_libnuma_install_plan(os_id: &str, id_like: &str) -> SystemPackageIn } } -/// The real host's `/etc/os-release`, falling back to `/usr/lib/os-release` -/// when the former is absent (per `os-release(5)`). -/// -/// Not routed through [`crate::host_path`]: the hints name package-manager -/// commands for the real machine, so a simulated host root must not choose -/// their distribution. -#[cfg(target_os = "linux")] -fn real_os_release() -> crate::hardware_root::OsRelease { - crate::hardware_root::read_os_release(crate::hardware_root::identity_path_for_str) -} - const OPENMPI_MANUAL_HINT: &str = "install your distribution's OpenMPI runtime package (providing libmpi.so / libmpi_cxx.so and mpirun)"; const LIBATOMIC_MANUAL_HINT: &str = "install your distribution's libatomic runtime package (providing libatomic.so.1)"; @@ -738,7 +727,7 @@ pub fn install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_openmpi_install_plan, OPENMPI_MANUAL_HINT, ) @@ -765,7 +754,7 @@ pub fn libatomic_install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_libatomic_install_plan, LIBATOMIC_MANUAL_HINT, ) @@ -792,7 +781,7 @@ pub fn libnuma_install_hint() -> InstallHint { #[cfg(target_os = "linux")] { install_hint_for_read( - &real_os_release(), + &crate::hardware_root::real_os_release(), build_libnuma_install_plan, LIBNUMA_MANUAL_HINT, ) @@ -1012,10 +1001,9 @@ mod tests { } /// `install_hint_for_read`/`plan_for_os_release_read` are the one seam - /// `openmpi::real_os_release()` and `system_packages::real_os_release()` - /// hand their result to — both now one-line delegations to - /// `hardware_root::read_os_release`, so reverting either back to a raw - /// `/etc/os-release` read is a type error here, not a silent regression. + /// `hardware_root::real_os_release()` hands its result to, from here and + /// from every caller in the `rocm` binary, so reverting any of them back to + /// a raw `/etc/os-release` read is a type error, not a silent regression. /// This still pins the behaviour at the seam itself: an `OsRelease` that /// says the fallback path was read produces the same plan and hint a /// primary-path read with the same content would. diff --git a/docs/architecture.md b/docs/architecture.md index 0c789a98e5..a952da9dcd 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -43,7 +43,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`dash.rs`](../apps/rocm/src/dash.rs) — full domain extraction. - [`dash_seam.rs`](../apps/rocm/src/dash_seam.rs) — full domain extraction. - [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: the command handlers for `rocm diagnose` and `rocm fix` (`diagnose()`, `fix()`) and the report step behind `rocm diagnose --report` (`show_prepared_report`, `perform_delivery`, `established_entry`). `diagnose()` and `fix()` are `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`; the other three functions are private. `diagnose()` gets its `--model` verdict from [`model_fit.rs`](../apps/rocm/src/model_fit.rs), and `fix()` returns a failing exit code through `FixExitCode`, which stays in `main.rs` with the other exit-code plumbing. The report and delivery regression tests live in this module's own `mod tests`; the exit-code tests stay in `main.rs`. -- [`driver_install.rs`](../apps/rocm/src/driver_install.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6a split, ROCMAI-82; see the Phase 6a variant note above): `install_driver()`/`reconcile_driver_install()`, running an approved plan's shell commands, and the install state persisted to `state.json` (`DriverInstallState`, `DriverReconciliationState` and the passive-check types). The plan it runs lives in [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs), which this module imports from and which does not import from it; the os-release the plan is built from comes from `real_os_release` in [`system_packages.rs`](../apps/rocm/src/system_packages.rs). Its `pub(crate)` surface is `install_driver` and `reconcile_driver_install`, `DriverInstallResult` and `DriverInstallError` (read from `install_cmd.rs`'s `install()`), `driver_install_state_path`, and a re-export of `host_install_driver_only_reports` for [`chat.rs`](../apps/rocm/src/chat.rs) and [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs). +- [`driver_install.rs`](../apps/rocm/src/driver_install.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6a split, ROCMAI-82; see the Phase 6a variant note above): `install_driver()`/`reconcile_driver_install()`, running an approved plan's shell commands, and the install state persisted to `state.json` (`DriverInstallState`, `DriverReconciliationState` and the passive-check types). The plan it runs lives in [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs), which this module imports from and which does not import from it; the os-release the plan is built from comes from `real_os_release` in [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), the one reader of the real host's os-release. Its `pub(crate)` surface is `install_driver` and `reconcile_driver_install`, `DriverInstallResult` and `DriverInstallError` (read from `install_cmd.rs`'s `install()`), `driver_install_state_path`, and a re-export of `host_install_driver_only_reports` for [`chat.rs`](../apps/rocm/src/chat.rs) and [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs). - [`driver_plan.rs`](../apps/rocm/src/driver_plan.rs) — full domain extraction, crate-level (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the plan `rocm install driver` builds and prints. It owns the model (`DriverInstallPlan` with `execution_commands`, `DriverPlanCommand`, `DriverCommandPhase`, and `PrivilegeEscalation`, which decides whether a generated shell line gets a `sudo ` prefix), `render_driver_install_plan` with its approval label, the helpers the builders share (`driver_command`, `driver_root_preflight_checks`, `resolve_shell_default_template`, `is_safe_shell_token`), and the builders: `build_driver_install_plan_read`, which `install_driver` calls with the real host's os-release, and `build_driver_install_plan_at` beneath it, with the apt, dnf and SLES plans, the `ID_LIKE` fallback, the repository URL helpers, `resolve_repo_version_expr` (the `ROCM_CLI_AMDGPU_DRIVER_VERSION` override and the retired version variables it refuses), and the refusal plans for an unreadable os-release or an invalid override. It also owns `install_driver_only_reports_on`/`host_install_driver_only_reports` (true on native Windows, where the plan only reports); `driver_install.rs` re-exports the latter. It declares the child module [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) and re-exports its `wsl_rocdxg_driver_plan`. Production code in this file uses only `std` and `rocm_core`, and nothing here depends on `driver_install.rs`. The model types and their fields, `execution_commands`, `PrivilegeEscalation::detect`, `driver_command`, `render_driver_install_plan`, `build_driver_install_plan_read`, `install_driver_only_reports_on` and `host_install_driver_only_reports` are `pub(crate)`, as is `invalid_driver_version_override_plan`, which a test that stays in `driver_install.rs` builds a plan with; the rest is private. Its test module is `pub(crate)` so `driver_install.rs`'s tests can reuse its `test_examine` and `os_release_found` helpers. It holds the plan-builder tests, including the one that keeps the supported-distro matrix in step with `dkms_driver_plan_is_supported` in `rocm-core`, and the tests of the template resolver and the shell-token filter. - [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) — full domain extraction, a child of `driver_plan.rs` that depends on the plan model, `driver_command`, `driver_root_preflight_checks` and `resolve_shell_default_template` in its parent, and on `truthy_env` in [`therock.rs`](../apps/rocm/src/therock.rs) (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the WSL2 rocDXG plan. `wsl_rocdxg_driver_plan` builds the plan that installs the `rocdxg-roct` package, `resolve_rocdxg_verification` decides how that download is authenticated (a supplied digest, the digest pinned for the version in `ROCDXG_PINNED_DIGESTS`, or the explicit opt-out), and `wsl_rocdxg_refusal_plan` is the plan that installs nothing and says why when it cannot be authenticated or the version is not a well-formed package version. `ROCDXG_PINNED_DIGESTS` is the trust anchor an unattended root install is checked against, so a change to it is a security-relevant diff; it moved byte for byte. `wsl_rocdxg_driver_plan` is `pub(crate)` and re-exported by `driver_plan.rs` (its builders call it, and tests in `driver_install.rs` use it as a plan fixture); everything else in production code is private. Its tests are the `wsl_rocdxg_*` plan tests, including the one that runs the generated digest step against a real file on Unix. The test-only `scoped_rocdxg_env` helper that clears the rocDXG environment lives here too, `pub(super)` for the plan-builder tests in `driver_plan.rs`. - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. @@ -86,7 +86,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` stays in [`main.rs`](../apps/rocm/src/main.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. -- [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and `real_os_release`, the real host's os-release that every one of these plans is built from. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions and `real_os_release` are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), and [`driver_install.rs`](../apps/rocm/src/driver_install.rs) imports `real_os_release` from here directly. `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `real_os_release` ignores a simulated host root. +- [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and reads the os-release every one of these plans is built from through `rocm_core::hardware_root::real_os_release`, which has no copy here. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `rocm_core::hardware_root::real_os_release` and the three OpenMPI/libatomic/libnuma install hints ignore a simulated host root, and a scan that no source file of the binary reads os-release by hand. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) stays in [`main.rs`](../apps/rocm/src/main.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. - [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. @@ -208,7 +208,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs) — free-space preflight checks and out-of-space error reporting. - [`fake_executable.rs`](../crates/rocm-core/src/fake_executable.rs) — writing stand-in executables that tests across the workspace then run. - [`fix.rs`](../crates/rocm-core/src/fix.rs) — `rocm fix`: remediations for diagnosed failure modes. Its `RECIPES` list (in [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs)), together with the checkers under `diagnose/` (their table is [`catalog.rs`](../crates/rocm-core/src/diagnose/catalog.rs)), is the authoritative failure catalog. [`fix.rs`](../crates/rocm-core/src/fix.rs) stays the module root, as [`examine.rs`](../crates/rocm-core/src/examine.rs) does for `examine/`, so links to it stay valid, with its children in `fix/`; it keeps the dispatch (`apply`, `Plan`, `plan_of`, `act_on`) and re-exports the names the rest of the crate and the apps use, so `crate::fix::exit`, `rocm_core::fix_exit`, `rocm_core::fix::FixClass` and the `crate::fix::` oracles that the tests under `diagnose/` call keep resolving. The children are private modules: [`fix/options.rs`](../crates/rocm-core/src/fix/options.rs) (the named exit codes, `FixOptions` and `FixClass`); [`fix/consent.rs`](../crates/rocm-core/src/fix/consent.rs) (the consent decision and `confirm`, the run and query timeouts, `current_os`, and the `fail!` helper, which it re-exports by path for its siblings); [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) (the host-mutating runners, one per auto-applicable fix, with their rc-file, PATH and device-pin helpers; only the four entry points the catalog names are visible to the root, and the `*_windows` variants are ordinary functions that every platform compiles); [`fix/catalog.rs`](../crates/rocm-core/src/fix/catalog.rs) (`FixRecipe`, the per-platform class table, `RECIPES`, the lookups by fix-id, and the `cfg(test)` oracles [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs) and the e2e crate hold their own copies to); [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) (`list_recipes`, a recipe's plan and verify lines, `format_flags`, and the refusal for a diagnosis position passed as a fix-id); and [`fix/manifest.rs`](../crates/rocm-core/src/fix/manifest.rs) (the published JSON contract, `CATALOG_CONTRACT_VERSION` and `catalog_manifest`). `render` and `manifest` read `RECIPES` from `catalog`; nothing in `catalog` reads either. -- [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs) — the root the hardware probes resolve host paths through (`host_path`): the real host in a normal build, a simulated host tree for the E2E suite in an `e2e-test-hooks` build. +- [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs) — the root the hardware probes resolve host paths through (`host_path`): the real host in a normal build, a simulated host tree for the E2E suite in an `e2e-test-hooks` build. It also owns `read_os_release` (the `/etc/os-release` read with its `/usr/lib/os-release` fallback) and `real_os_release`, the one reader every package-manager plan and install hint uses, which always reads the real host even under a simulated root; the resolver it passes is private, so a second copy of that reader cannot be written outside this module. - [`log_tail.rs`](../crates/rocm-core/src/log_tail.rs) — `tail_lines`, the bounded log-tail read shared by the lemonade and vLLM adapters (no module doc comment). - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. diff --git a/docs/testing.md b/docs/testing.md index 97aa511110..24dfc6870a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2623,8 +2623,8 @@ rather than the real host. The `a_fallback_read_plans*`/ `an_unreadable_file*`/`distro_name_names_an_unreadable_file` tests in `rocm-core` and `rocm` construct an `OsRelease` directly — bypassing the real read entirely, since the production functions that do it -(`openmpi::real_os_release`, `system_packages::real_os_release`) -read the real machine +(`hardware_root::real_os_release`, which every caller goes through) +reads the real machine unconditionally and cannot honour a simulated root — to pin that a fallback-sourced read plans and hints identically to a primary-sourced one with the same text, and that a file which exists but cannot be read is named @@ -2645,6 +2645,19 @@ shell probe instead of merely leaving `$ID` unset, which this test does not exercise — see its own doc comment for why, and for the known limitation this leaves. +The package-manager side reads the real host through one function, +`hardware_root::real_os_release`. In the `rocm` binary's tests, +`system_packages::tests::real_os_release_ignores_a_simulated_host_root` +(`--features e2e-test-hooks`, Linux) plants a root naming another distribution +and expects the real file back, and +`system_packages::tests::the_install_hints_read_the_real_host_and_ignore_a_simulated_root` +holds the three install hints to the same rule. Both live in that process +rather than in `rocm-core`'s, so the environment variable they set cannot +overlap a `rocm-core` test that reads host paths. +`system_packages::tests::no_caller_in_this_binary_reads_the_real_os_release_by_hand` +fails when a source file of the `rocm` binary reads the file with its own +resolver instead of calling it. + `rocm examine` reads the file through the simulated host root (see Simulated Host Root in [release-trust.md](release-trust.md)), so scenarios `examine-20` to `examine-23` and `examine-27` to `examine-28` in diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index aa182ec62f..c9d4b9f3d2 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -286,9 +286,9 @@ Feature: GPU detection and system inspection # machine other than the runner. The driver plan cannot be exercised this way: # `rocm install driver` builds a plan that runs against the real machine, so it # deliberately reads the real `/etc/os-release` (`real_os_release` in - # `system_packages.rs`, whose test there pins that it ignores a simulated - # host root); the plan built from it is covered by the unit tests in - # `driver_plan.rs`. + # `hardware_root.rs`, whose test in `system_packages.rs` pins that it ignores + # a simulated host root); the plan built from it is covered by the unit tests + # in `driver_plan.rs`. # # The planted distributions are ones no CI runner or developer host here runs, # so a probe that read the real file instead would report a different one. diff --git a/tests/e2e-cucumber/tests/e2e/driver_steps.rs b/tests/e2e-cucumber/tests/e2e/driver_steps.rs index a1532cc998..cdb0d7debf 100644 --- a/tests/e2e-cucumber/tests/e2e/driver_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/driver_steps.rs @@ -41,10 +41,9 @@ async fn assert_driver_plan_matches_host_release(world: &mut E2eWorld) { // breaks the arm-selection logic pass by construction. The hosted // `ubuntu-latest` E2E image (26.04 once GitHub's migration completes) and the self-hosted GPU // lane (24.04) both take the apt arm, with their own release codename. - let os_release_text = - rocm_core::hardware_root::read_os_release(rocm_core::hardware_root::identity_path_for_str) - .text() - .to_owned(); + let os_release_text = rocm_core::hardware_root::real_os_release() + .text() + .to_owned(); let Ok(os_release) = rocm_core::os_release::parse(&os_release_text) else { panic!("host's /etc/os-release is unreadable:\n{output}"); }; From 9009f34c2a0ea545ab98f1b41a04a0e259b06c31 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 20:24:09 +0000 Subject: [PATCH 10/20] test: pin which manifest files each way of registering a runtime writes A runtime has up to two manifest files: its registry entry and the .rocm-cli-runtime.json that ROCm CLI keeps in the install root. Installing writes both and always overwrites. Importing and adopting write the registry entry only and refuse to replace an existing entry unless asked, and recovering the runtime that setup created restores a missing registry entry from the manifest already in its folder. Nothing asserted that file set per path. These tests assert, for install, import, adopt and setup recovery, which of the two files exist afterwards, and for import and adopt that a duplicate is refused (message and untouched entry together) until replacement is requested. They pass on the current code, so they pin its behaviour before the writers are merged. The import scenario gains an assertion that the registry holds the entry and that the imported folder gains no manifest file, after the first import and after a replacing one. The adopt test's environment fixture moves into a helper so the new adopt test reuses it. Signed-off-by: Jussi Elo --- apps/rocm/src/runtimes_cmd.rs | 275 +++++++++++++++--- .../features/runtime_lifecycle.feature | 2 + .../tests/e2e/runtime_lifecycle_steps.rs | 33 ++- 3 files changed, 274 insertions(+), 36 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index d27db770fd..d7dba7db29 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -869,7 +869,7 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> #[cfg(test)] pub(crate) mod tests { - use std::path::PathBuf; + use std::path::{Path, PathBuf}; use super::*; use crate::append_examine_runtime_state; @@ -2311,9 +2311,14 @@ pub(crate) mod tests { Ok(()) } - #[test] - fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { - let (root, paths) = test_paths("runtime-adopt"); + /// A Python environment outside ROCm CLI's folders that `rocm runtimes adopt` + /// accepts: an interpreter, the SDK libraries the probe names, and the probe + /// that reports them. Returns the environment folder, the request that adopts + /// it, and the probe. + fn adoptable_environment( + root: &Path, + replace: bool, + ) -> Result<(PathBuf, AdoptRuntimeRequest, therock::RocmSdkPythonProbe)> { let external_root = root.join("external-therock-venv"); let scripts_dir = external_root.join(if cfg!(windows) { "Scripts" } else { "bin" }); let python_executable = scripts_dir.join(if cfg!(windows) { @@ -2342,37 +2347,43 @@ pub(crate) mod tests { fs::write(&amdhip, "amdhip")?; fs::write(&hipblas, "hipblas")?; - let adopted = adopt_runtime_from_probe( - &paths, - AdoptRuntimeRequest { - python_executable, - install_root: external_root.clone(), - runtime_id: "therock-release:gfx120X-all".to_owned(), - runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), - replace: false, - }, - therock::RocmSdkPythonProbe { - import_ok: true, - rocm_sdk_version: Some("7.13.0".to_owned()), - root_path: Some(sdk_root.clone()), - bin_path: Some(sdk_bin.clone()), - runtime_roots: vec![sdk_root], - bin_paths: vec![sdk_bin.clone()], - library_paths: vec![sdk_bin], - resolved_libraries: vec![ - therock::RocmSdkLibraryProbe { - shortname: "amdhip64".to_owned(), - paths: vec![amdhip], - }, - therock::RocmSdkLibraryProbe { - shortname: "hipblas".to_owned(), - paths: vec![hipblas], - }, - ], - resolved_target_family: Some("gfx120X-all".to_owned()), - ..therock::RocmSdkPythonProbe::default() - }, - )?; + let request = AdoptRuntimeRequest { + python_executable, + install_root: external_root.clone(), + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: "adopted-release-pip-gfx120x-all-7-13-0".to_owned(), + replace, + }; + let probe = therock::RocmSdkPythonProbe { + import_ok: true, + rocm_sdk_version: Some("7.13.0".to_owned()), + root_path: Some(sdk_root.clone()), + bin_path: Some(sdk_bin.clone()), + runtime_roots: vec![sdk_root], + bin_paths: vec![sdk_bin.clone()], + library_paths: vec![sdk_bin], + resolved_libraries: vec![ + therock::RocmSdkLibraryProbe { + shortname: "amdhip64".to_owned(), + paths: vec![amdhip], + }, + therock::RocmSdkLibraryProbe { + shortname: "hipblas".to_owned(), + paths: vec![hipblas], + }, + ], + resolved_target_family: Some("gfx120X-all".to_owned()), + ..therock::RocmSdkPythonProbe::default() + }; + Ok((external_root, request, probe)) + } + + #[test] + fn runtime_adopt_records_read_only_manifest_from_probe() -> Result<()> { + let (root, paths) = test_paths("runtime-adopt"); + let (external_root, request, probe) = adoptable_environment(&root, false)?; + + let adopted = adopt_runtime_from_probe(&paths, request, probe)?; assert!(adopted.read_only); assert_eq!(adopted.channel, "release"); @@ -2407,6 +2418,200 @@ pub(crate) mod tests { Ok(()) } + /// Which of the two files a registered runtime can have exist: its registry + /// entry, and the `.rocm-cli-runtime.json` ROCm CLI keeps in the install + /// root as proof that it owns the folder. + fn registered_files(paths: &AppPaths, install_root: &Path, runtime_key: &str) -> (bool, bool) { + ( + runtime_manifest_path(paths, runtime_key).is_file(), + install_root.join(".rocm-cli-runtime.json").is_file(), + ) + } + + /// A runtime ROCm CLI installed itself, with both of its manifest files + /// deleted so a test sees only what the write under test creates. + fn installed_runtime_without_manifest_files( + paths: &AppPaths, + ) -> Result { + let manifest = write_test_pip_runtime( + paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(runtime_manifest_path(paths, &manifest.runtime_key))?; + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; + assert_eq!( + registered_files(paths, &manifest.install_root, &manifest.runtime_key), + (false, false) + ); + Ok(manifest) + } + + /// Registering a runtime whose registry entry exists is refused until the + /// caller asks to replace it. `register(replace)` makes the attempt; the + /// message and the entry on disk are asserted together, and the same call + /// with `replace` set is what clears the refusal. + fn assert_duplicate_registration_needs_replace( + paths: &AppPaths, + runtime_key: &str, + register: impl Fn(bool) -> Result<()>, + ) -> Result<()> { + let entry = runtime_manifest_path(paths, runtime_key); + fs::write(&entry, "stale entry")?; + + let error = register(false).unwrap_err().to_string(); + + assert!( + error.contains("already exists") && error.contains("--replace"), + "{error}" + ); + assert_eq!( + fs::read_to_string(&entry)?, + "stale entry", + "a refused registration must leave the existing entry alone" + ); + register(true)?; + let replaced: therock::InstalledRuntimeManifest = + serde_json::from_slice(&fs::read(&entry)?)?; + assert_eq!(replaced.runtime_key, runtime_key); + Ok(()) + } + + /// Installing writes both files, and a second install of the same key + /// overwrites both without being asked: it is the same runtime again. + #[test] + fn runtime_install_writes_the_registry_entry_and_the_install_root_manifest() -> Result<()> { + let (root, paths) = test_paths("runtime-files-install"); + let manifest = installed_runtime_without_manifest_files(&paths)?; + + crate::runtime_registry::save_runtime_manifest(&paths, &manifest)?; + + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, true) + ); + let reinstalled = therock::InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest.clone() + }; + crate::runtime_registry::save_runtime_manifest(&paths, &reinstalled)?; + for path in [ + runtime_manifest_path(&paths, &manifest.runtime_key), + manifest.install_root.join(".rocm-cli-runtime.json"), + ] { + let written: therock::InstalledRuntimeManifest = + serde_json::from_slice(&fs::read(&path)?)?; + assert_eq!(written.version, "7.14.0", "{}", path.display()); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Importing registers a folder somebody else owns, so it writes the + /// registry entry and nothing into that folder, and refuses to replace an + /// existing entry unless asked. + #[test] + fn runtime_import_writes_the_registry_entry_only_and_refuses_a_duplicate() -> Result<()> { + let (root, paths) = test_paths("runtime-files-import"); + let manifest = installed_runtime_without_manifest_files(&paths)?; + let exported_manifest = root.join("external-runtime.json"); + fs::write(&exported_manifest, serde_json::to_vec_pretty(&manifest)?)?; + + import_runtime_manifest(&paths, &exported_manifest, false)?; + + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, false) + ); + assert_duplicate_registration_needs_replace(&paths, &manifest.runtime_key, |replace| { + import_runtime_manifest(&paths, &exported_manifest, replace).map(drop) + })?; + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, false), + "replacing an import must not start writing into the folder either" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Adopting has the same contract as importing: registry entry only, and a + /// duplicate needs `--replace`. + #[test] + fn runtime_adopt_writes_the_registry_entry_only_and_refuses_a_duplicate() -> Result<()> { + let (root, paths) = test_paths("runtime-files-adopt"); + let (external_root, request, probe) = adoptable_environment(&root, false)?; + let runtime_key = request.runtime_key.clone(); + + adopt_runtime_from_probe(&paths, request.clone(), probe.clone())?; + + assert_eq!( + registered_files(&paths, &external_root, &runtime_key), + (true, false) + ); + assert_duplicate_registration_needs_replace(&paths, &runtime_key, |replace| { + let request = AdoptRuntimeRequest { + replace, + ..request.clone() + }; + adopt_runtime_from_probe(&paths, request, probe.clone()).map(drop) + })?; + assert_eq!( + registered_files(&paths, &external_root, &runtime_key), + (true, false), + "replacing an adoption must not start writing into the environment either" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Recovering the runtime `rocm setup` created restores its registry entry + /// from the manifest in its folder, leaves that manifest in place, and does + /// not replace an entry that is already there. + #[test] + fn setup_recovery_restores_the_registry_entry_and_keeps_the_install_root_manifest() -> Result<()> + { + let (root, paths) = test_paths("runtime-files-recovery"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + fs::remove_file(&entry)?; + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (false, true) + ); + + let recovered = recover_setup_runtime_registration(&paths, &config)?; + + assert_eq!(recovered.as_deref(), Some(manifest.runtime_key.as_str())); + assert_eq!( + registered_files(&paths, &manifest.install_root, &manifest.runtime_key), + (true, true) + ); + fs::write(&entry, "entry written since")?; + recover_setup_runtime_registration(&paths, &config)?; + assert_eq!( + fs::read_to_string(&entry)?, + "entry written since", + "recovery only fills a missing entry" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn runtime_uninstall_removes_managed_root_and_clears_active_state() -> Result<()> { let (root, paths) = test_paths("runtime-uninstall-managed"); diff --git a/tests/e2e-cucumber/features/runtime_lifecycle.feature b/tests/e2e-cucumber/features/runtime_lifecycle.feature index e5e9537edf..5f8da06c42 100644 --- a/tests/e2e-cucumber/features/runtime_lifecycle.feature +++ b/tests/e2e-cucumber/features/runtime_lifecycle.feature @@ -48,10 +48,12 @@ Feature: Runtime lifecycle state machine Given a runtime manifest to import When the user imports the runtime Then the runtime is registered as read-only + And the registry records the import and its folder gains no manifest file When the user imports the same runtime again Then the CLI refuses because it already exists When the user imports it again allowing replacement Then the import succeeds + And the registry records the import and its folder gains no manifest file # `rocm runtimes list` prefixes each row with `*`/`-`/a blank to mark active, # rollback-target, and neither, with nothing else on the page explaining what diff --git a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs index c127245530..1838780c1e 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_lifecycle_steps.rs @@ -273,9 +273,19 @@ fn active_runtime_key(world: &E2eWorld) -> String { /// Path to an importable manifest file (not yet in the registry) for the import /// scenario, with its install_root created so the import validates. +/// The folder the manifest to import points at: a runtime somebody else owns. +fn import_install_root(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("runtime-import") +} + fn write_import_manifest(world: &E2eWorld) -> PathBuf { let root = world.isolated_root.as_ref().expect("no isolated root"); - let install_root = root.path().join("runtime-import"); + let install_root = import_install_root(world); std::fs::create_dir_all(&install_root).expect("failed to create import install root"); std::fs::write(install_root.join("payload.txt"), "payload") .expect("failed to write import payload"); @@ -740,6 +750,27 @@ async fn imported_readonly(world: &mut E2eWorld) { ); } +/// What an import creates on disk: the registry entry, and nothing in the +/// folder it points at. The manifest ROCm CLI keeps in an install root is its +/// proof that it owns the folder, and an imported runtime is somebody else's. +/// Unit tests cover the same file set for adopt and setup recovery, which no +/// scenario reaches without a Python environment to probe. +#[then("the registry records the import and its folder gains no manifest file")] +async fn import_writes_registry_entry_only(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{IMPORT_KEY}.json")); + assert!( + entry.is_file(), + "expected a registry entry at {}", + entry.display() + ); + let sidecar = import_install_root(world).join(".rocm-cli-runtime.json"); + assert!( + !sidecar.exists(), + "an import must not write into the folder it registers: {}", + sidecar.display() + ); +} + #[then("the CLI refuses because it already exists")] async fn import_duplicate_refused(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no command rc recorded"); From 78c6171ebd404912de8503c89a797fd3ceafad92 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 20:55:13 +0000 Subject: [PATCH 11/20] refactor: register every runtime through one manifest writer Two functions wrote a runtime's manifests. The SDK install path wrote the registry entry and the .rocm-cli-runtime.json in the install root and always overwrote; the import, adopt and setup-recovery path wrote only the registry entry and refused to overwrite unless --replace was given. Which files a runtime got depended on which function its caller happened to use, and nothing tied the choice to the thing it stands for. write_runtime_manifest in runtime_registry.rs is now the only writer. It takes the overwrite policy (ManifestOverwrite::Replace or ::Refuse) as its one parameter and decides the install-root file from InstalledRuntimeManifest::owns_install_root: a runtime ROCm CLI installed gets it, an imported or adopted one, which is someone else's folder that uninstall and prune never delete, does not. The private writer in runtime_adopt.rs is gone. The file is left alone when it already holds what would be written, so recovering a runtime from its own manifest does not rewrite it. No user-visible file set changes: install still writes both files and replaces, import and adopt still write the registry entry only and refuse a duplicate without --replace, recovery still only fills a missing entry. The test that pins this per path stays green across the change, and new tests tie the writer's ownership rule to uninstall's, check each policy and the skipped rewrite, and fail if production code outside runtime_registry.rs opens a registry entry for writing itself. Neither file is written atomically; that is left to the shared atomic writer work. Signed-off-by: Jussi Elo --- apps/rocm/src/runtime_adopt.rs | 45 ++--- apps/rocm/src/runtime_registry.rs | 299 ++++++++++++++++++++++++++++-- apps/rocm/src/runtimes_cmd.rs | 44 ++++- apps/rocm/src/therock.rs | 7 +- docs/architecture.md | 2 +- docs/testing.md | 19 ++ 6 files changed, 360 insertions(+), 56 deletions(-) diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 549753818a..393e509d17 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -22,8 +22,8 @@ use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_executable_in_env}; use crate::runtime_registry::{ - paths_equivalent, runtime_manifest_path, runtime_registry_dir, - validate_runtime_manifest_for_activation, + ManifestOverwrite, paths_equivalent, runtime_manifest_path, runtime_registry_dir, + validate_runtime_manifest_for_activation, write_runtime_manifest, }; use crate::therock; @@ -58,7 +58,11 @@ pub(crate) fn import_runtime_manifest( validate_runtime_manifest_for_activation(&manifest) .with_context(|| format!("imported runtime `{}` is not usable", manifest.runtime_key))?; - write_runtime_registry_manifest(paths, &manifest, replace)?; + write_runtime_manifest( + paths, + &manifest, + ManifestOverwrite::for_replace_flag(replace), + )?; Ok(manifest) } @@ -314,7 +318,11 @@ pub(crate) fn adopt_runtime_from_probe( }; validate_runtime_manifest_for_activation(&manifest) .with_context(|| format!("adopted runtime `{}` is not usable", manifest.runtime_key))?; - write_runtime_registry_manifest(paths, &manifest, request.replace)?; + write_runtime_manifest( + paths, + &manifest, + ManifestOverwrite::for_replace_flag(request.replace), + )?; Ok(manifest) } @@ -350,33 +358,6 @@ fn parse_therock_runtime_id(runtime_id: &str) -> Result<(String, String)> { Ok((channel.to_owned(), family.to_owned())) } -fn write_runtime_registry_manifest( - paths: &AppPaths, - manifest: &therock::InstalledRuntimeManifest, - replace: bool, -) -> Result<()> { - let manifest = manifest.clone().normalize_storage_paths(); - let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); - if registry_path.exists() && !replace { - bail!( - "runtime registry entry already exists: {}; pass --replace to overwrite it", - registry_path.display() - ); - } - fs::create_dir_all( - registry_path - .parent() - .context("runtime registry path has no parent directory")?, - )?; - fs::write( - ®istry_path, - serde_json::to_vec_pretty(&manifest) - .context("failed to serialize runtime registry manifest")?, - ) - .with_context(|| format!("failed to write {}", registry_path.display()))?; - Ok(()) -} - pub(crate) fn recover_setup_runtime_registration( paths: &AppPaths, config: &RocmCliConfig, @@ -425,7 +406,7 @@ pub(crate) fn recover_setup_runtime_registration( })?; if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { - write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { + write_runtime_manifest(paths, &manifest, ManifestOverwrite::Refuse).with_context(|| { format!( "failed to restore setup runtime `{}` into {}", manifest.runtime_key, diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 9f9fa17ce7..5477674dde 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -121,6 +121,15 @@ impl InstalledRuntimeManifest { wheel_composition_includes_devel(self.wheel_composition.as_ref()).unwrap_or(self.devel) } + /// Whether ROCm CLI owns the folder this runtime lives in: it installed it, + /// rather than importing or adopting somebody else's. Ownership is what + /// decides whether the install root carries a `.rocm-cli-runtime.json` (see + /// [`write_runtime_manifest`]) and whether uninstall and prune may delete + /// the folder, so those cannot disagree about which runtimes are whose. + pub(crate) const fn owns_install_root(&self) -> bool { + !self.read_only && self.imported_from.is_none() + } + fn normalize_host_paths(mut self) -> Self { self.install_root = normalize_manifest_path(self.install_root); self.python_launcher = self @@ -336,30 +345,79 @@ pub(crate) fn runtime_manifest_path(paths: &AppPaths, runtime_key: &str) -> Path runtime_registry_dir(paths).join(format!("{runtime_key}.json")) } -pub(crate) fn save_runtime_manifest( +/// What [`write_runtime_manifest`] does when the registry already holds an entry +/// under the manifest's key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ManifestOverwrite { + /// Write over it. An install of a runtime ROCm CLI already installed is the + /// same runtime again. + Replace, + /// Refuse, and name `--replace`. Importing or adopting a runtime must not + /// displace a registration the user made earlier unless they ask to. + Refuse, +} + +impl ManifestOverwrite { + /// The policy for a command with a `--replace` flag. + pub(crate) const fn for_replace_flag(replace: bool) -> Self { + if replace { Self::Replace } else { Self::Refuse } + } +} + +/// Write the manifest files of a runtime: the registry entry, and, for a +/// runtime ROCm CLI owns ([`InstalledRuntimeManifest::owns_install_root`]), the +/// `.rocm-cli-runtime.json` in its install root. +/// +/// Every way a runtime gets registered goes through this one routine: an SDK +/// install, `rocm runtimes import`, `rocm runtimes adopt`, and the recovery of +/// the runtime `rocm setup` created. `overwrite` is the only thing a caller +/// chooses. +/// +/// The file in the install root is ROCm CLI's proof that it owns the folder: +/// uninstall and prune delete a folder only when it carries one that matches +/// the registry, and activation requires one of a runtime that is not +/// read-only. An imported or adopted runtime is somebody else's folder, which +/// uninstall and prune never delete and activation accepts without one, so it +/// gets none: a root the user may not be able to write to must not make +/// registering it fail, and an import's source manifest may be that very file +/// in another install. +/// The file is not rewritten when it already holds exactly what would be +/// written, so recovering a runtime from its own manifest leaves it alone. +/// +/// Neither file is written atomically; that is a separate piece of work shared +/// with the other writers in the workspace. +pub(crate) fn write_runtime_manifest( paths: &AppPaths, manifest: &InstalledRuntimeManifest, + overwrite: ManifestOverwrite, ) -> Result<()> { let manifest = manifest.clone().normalize_storage_paths(); let registry_path = runtime_manifest_path(paths, &manifest.runtime_key); + if overwrite == ManifestOverwrite::Refuse && registry_path.exists() { + bail!( + "runtime registry entry already exists: {}; pass --replace to overwrite it", + registry_path.display() + ); + } fs::create_dir_all( registry_path .parent() .context("runtime manifest registry path has no parent directory")?, )?; - fs::write( - ®istry_path, - serde_json::to_vec_pretty(&manifest).context("failed to serialize runtime manifest")?, - ) - .with_context(|| format!("failed to write {}", registry_path.display()))?; - - let local_manifest_path = manifest.install_root.join(".rocm-cli-runtime.json"); - fs::write( - &local_manifest_path, - serde_json::to_vec_pretty(&manifest) - .context("failed to serialize local runtime manifest")?, - ) - .with_context(|| format!("failed to write {}", local_manifest_path.display()))?; + let encoded = + serde_json::to_vec_pretty(&manifest).context("failed to serialize runtime manifest")?; + fs::write(®istry_path, &encoded) + .with_context(|| format!("failed to write {}", registry_path.display()))?; + + if manifest.owns_install_root() { + let local_manifest_path = manifest.install_root.join(".rocm-cli-runtime.json"); + let already_current = + fs::read(&local_manifest_path).is_ok_and(|existing| existing == encoded); + if !already_current { + fs::write(&local_manifest_path, &encoded) + .with_context(|| format!("failed to write {}", local_manifest_path.display()))?; + } + } Ok(()) } @@ -791,13 +849,13 @@ mod tests { /// that neither `rocm runtimes list` nor `rocm examine` surfaces `devel`, /// so the state is invisible. /// - /// LIMIT: this covers `save_runtime_manifest` only. It does NOT reach + /// LIMIT: this covers `write_runtime_manifest` only. It does NOT reach /// `install_sdk`, which is what actually threads `include_devel` into the /// manifest — that call needs `uv`, a live index, and a real `rocm_sdk` /// probe, so it has no unit coverage here. Hardcoding `devel: true` at that /// write site passes this test and the rest of the suite. #[test] - fn save_runtime_manifest_replaces_devel_rather_than_merging() -> Result<()> { + fn write_runtime_manifest_replaces_devel_rather_than_merging() -> Result<()> { let (root, paths) = test_paths("manifest-replaces-devel"); let install_root = root.join("install-root"); fs::create_dir_all(&install_root)?; @@ -813,14 +871,14 @@ mod tests { with_devel.devel, "test fixture should start with devel: true" ); - save_runtime_manifest(&paths, &with_devel)?; + write_runtime_manifest(&paths, &with_devel, ManifestOverwrite::Replace)?; let without_devel = InstalledRuntimeManifest { devel: false, installed_at_unix_ms: 2, ..with_devel.clone() }; - save_runtime_manifest(&paths, &without_devel)?; + write_runtime_manifest(&paths, &without_devel, ManifestOverwrite::Replace)?; let manifests = load_runtime_manifests(&paths)?; let reloaded = manifests @@ -836,6 +894,211 @@ mod tests { Ok(()) } + /// A manifest for `runtime_key` whose install root is a folder under `root`. + fn manifest_in( + root: &Path, + runtime_key: &str, + read_only: bool, + imported_from: Option, + ) -> Result { + let install_root = root.join(runtime_key); + fs::create_dir_all(&install_root)?; + Ok(InstalledRuntimeManifest { + install_root, + read_only, + imported_from, + ..test_runtime_manifest(runtime_key, "therock-release:gfx120X-all", 1) + }) + } + + /// The policy is the only thing a caller chooses, and it decides one thing: + /// whether an existing registry entry is replaced or left alone. The refusal + /// is asserted as message and state together, and with the entry still the + /// old bytes, because a refusal that wrote first would print the same text. + #[test] + fn write_runtime_manifest_refuses_an_existing_entry_only_under_the_refuse_policy() -> Result<()> + { + let (root, paths) = test_paths("manifest-overwrite-policy"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Refuse)?; + assert!( + entry.is_file(), + "nothing registered yet, so nothing to refuse" + ); + + fs::write(&entry, "stale entry")?; + let error = write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Refuse) + .unwrap_err() + .to_string(); + assert!( + error.contains("already exists") && error.contains("--replace"), + "{error}" + ); + assert_eq!(fs::read_to_string(&entry)?, "stale entry"); + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + let replaced: InstalledRuntimeManifest = serde_json::from_slice(&fs::read(&entry)?)?; + assert_eq!(replaced.runtime_key, manifest.runtime_key); + + assert_eq!( + ManifestOverwrite::for_replace_flag(true), + ManifestOverwrite::Replace + ); + assert_eq!( + ManifestOverwrite::for_replace_flag(false), + ManifestOverwrite::Refuse + ); + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// Which manifests get a `.rocm-cli-runtime.json` in the install root: + /// exactly the ones ROCm CLI owns. A read-only runtime and an imported one + /// (the two ways a runtime is not ROCm CLI's) get a registry entry and leave + /// their folder alone, whichever policy registered them. + #[test] + fn write_runtime_manifest_writes_the_install_root_manifest_only_for_an_owned_runtime() + -> Result<()> { + let (root, paths) = test_paths("manifest-install-root-file"); + let cases = [ + ("owned", false, None, true), + ("read-only", true, None, false), + ( + "imported", + false, + Some(PathBuf::from("elsewhere/manifest.json")), + false, + ), + ]; + for overwrite in [ManifestOverwrite::Replace, ManifestOverwrite::Refuse] { + for (label, read_only, imported_from, gets_the_file) in &cases { + let key = format!("{label}-{overwrite:?}"); + let manifest = manifest_in(&root, &key, *read_only, imported_from.clone())?; + + write_runtime_manifest(&paths, &manifest, overwrite)?; + + assert_eq!( + manifest.owns_install_root(), + *gets_the_file, + "{label} / {overwrite:?}" + ); + assert!( + runtime_manifest_path(&paths, &key).is_file(), + "{label} / {overwrite:?}: registry entry" + ); + assert_eq!( + manifest + .install_root + .join(".rocm-cli-runtime.json") + .is_file(), + *gets_the_file, + "{label} / {overwrite:?}: install root manifest" + ); + } + } + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// Recovering a runtime from its own install-root manifest must not touch + /// that file when it already holds what would be written, so a setup folder + /// the user cannot write to does not turn a recovery into an error. The file + /// is made read-only: a write would fail where a skip succeeds. (Not + /// asserted when the file stays writable regardless, as for an + /// administrator.) + #[test] + #[allow(clippy::permissions_set_readonly_false)] + fn write_runtime_manifest_leaves_a_current_install_root_manifest_alone() -> Result<()> { + let (root, paths) = test_paths("manifest-current-install-root-file"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + let local = manifest.install_root.join(".rocm-cli-runtime.json"); + let mut permissions = fs::metadata(&local)?.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&local, permissions.clone())?; + let still_writable = fs::OpenOptions::new().append(true).open(&local).is_ok(); + + let unchanged = write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace); + let changed = InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest + }; + let rewritten = write_runtime_manifest(&paths, &changed, ManifestOverwrite::Replace); + + permissions.set_readonly(false); + fs::set_permissions(&local, permissions)?; + unchanged.context("an install root manifest that is already current is not rewritten")?; + if !still_writable { + assert!( + rewritten.is_err(), + "a different manifest is written, which the read-only file refuses" + ); + } + let _ = fs::remove_dir_all(&root); + Ok(()) + } + + /// Every way of registering a runtime goes through + /// [`write_runtime_manifest`]. A second writer would be free to drift again + /// (a different overwrite rule, a missing install-root manifest), so no + /// production code outside this file may open a registry entry for writing. + /// This reads the sources: the writers differ only in how they are called, + /// which no behavioural test of one caller can see in another. + #[test] + fn only_the_shared_routine_writes_a_registry_entry() { + fn production_part(source: &str) -> &str { + [ + "#[cfg(test)]\nmod tests", + "#[cfg(test)]\npub(crate) mod tests", + ] + .iter() + .filter_map(|marker| source.find(marker)) + .min() + .map_or(source, |end| &source[..end]) + } + fn rust_files(dir: &Path, found: &mut Vec) { + for entry in fs::read_dir(dir).expect("source folder readable").flatten() { + let path = entry.path(); + if path.is_dir() { + rust_files(&path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") { + found.push(path); + } + } + } + let mut sources = Vec::new(); + rust_files( + &Path::new(env!("CARGO_MANIFEST_DIR")).join("src"), + &mut sources, + ); + assert!(sources.len() > 50, "the scan found the source tree"); + // This file holds the routine; `test_support.rs` is fixtures throughout. + let exempt = [ + Path::new(file!()).file_name(), + Some("test_support.rs".as_ref()), + ]; + for path in sources { + if exempt.contains(&path.file_name()) { + continue; + } + let source = fs::read_to_string(&path).expect("source readable"); + let production = production_part(&source); + for (at, _) in production.match_indices("fs::write(") { + let call = &production[at..(at + 200).min(production.len())]; + let call = call.split(';').next().unwrap_or(call); + assert!( + !call.contains("runtime_manifest_path(") + && !call.contains("registry_path") + && !call.contains("runtime_registry_dir("), + "{} writes a registry entry itself instead of calling write_runtime_manifest: {call}", + path.display() + ); + } + } + } + /// A manifest written before `devel` became opt-in has no such field, and /// those installs all had the toolchain. Reading one back must not claim /// otherwise, or the next update would silently strip it. diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index d7dba7db29..81fe8e301c 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -878,6 +878,7 @@ pub(crate) mod tests { use crate::runtime_install_finalize::{ finalize_successful_sdk_install, render_sdk_install_success, }; + use crate::runtime_registry::{ManifestOverwrite, write_runtime_manifest}; use crate::test_support::write_test_pip_runtime; use crate::tests::{ NEW_RUNTIME_KEY, OLD_RUNTIME_KEY, plant_service_record_on_runtime, test_paths, @@ -2486,7 +2487,7 @@ pub(crate) mod tests { let (root, paths) = test_paths("runtime-files-install"); let manifest = installed_runtime_without_manifest_files(&paths)?; - crate::runtime_registry::save_runtime_manifest(&paths, &manifest)?; + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; assert_eq!( registered_files(&paths, &manifest.install_root, &manifest.runtime_key), @@ -2496,7 +2497,7 @@ pub(crate) mod tests { version: "7.14.0".to_owned(), ..manifest.clone() }; - crate::runtime_registry::save_runtime_manifest(&paths, &reinstalled)?; + write_runtime_manifest(&paths, &reinstalled, ManifestOverwrite::Replace)?; for path in [ runtime_manifest_path(&paths, &manifest.runtime_key), manifest.install_root.join(".rocm-cli-runtime.json"), @@ -2612,6 +2613,45 @@ pub(crate) mod tests { Ok(()) } + /// The writer gives a runtime a manifest in its folder exactly when + /// uninstall, which prune asks the same question of, may delete that folder. + /// The two are written in different modules; this is what ties them. + #[test] + fn a_runtime_gets_an_install_root_manifest_exactly_when_uninstall_may_remove_its_folder() + -> Result<()> { + let (root, paths) = test_paths("runtime-files-ownership"); + for (label, read_only, imported) in [ + ("owned", false, false), + ("read-only", true, false), + ("imported", false, true), + ] { + let mut manifest = write_test_pip_runtime( + &paths, + &format!("release-pip-{label}"), + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + manifest.read_only = read_only; + manifest.imported_from = imported.then(|| PathBuf::from("elsewhere/manifest.json")); + fs::remove_file(manifest.install_root.join(".rocm-cli-runtime.json"))?; + + write_runtime_manifest(&paths, &manifest, ManifestOverwrite::Replace)?; + + let has_manifest = manifest + .install_root + .join(".rocm-cli-runtime.json") + .is_file(); + let may_remove = + should_remove_runtime_install_root(&manifest)? == InstallRootDecision::Remove; + assert_eq!(has_manifest, may_remove, "{label}"); + assert_eq!(has_manifest, manifest.owns_install_root(), "{label}"); + } + + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn runtime_uninstall_removes_managed_root_and_clears_active_state() -> Result<()> { let (root, paths) = test_paths("runtime-uninstall-managed"); diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 8efc17ebbe..50be71cfcd 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -72,7 +72,8 @@ pub(crate) use crate::runtime_registry::{ validate_rocm_sdk_runtime_probe, }; use crate::runtime_registry::{ - runtime_manifest_path, save_runtime_manifest, wheel_composition_includes_devel, + ManifestOverwrite, runtime_manifest_path, wheel_composition_includes_devel, + write_runtime_manifest, }; use crate::shell_quote::shell_quote; use anyhow::{Context, Result, bail}; @@ -735,7 +736,7 @@ fn install_wheel_runtime( devel: include_devel, installed_at_unix_ms: unix_time_millis(), }; - save_runtime_manifest(paths, &manifest)?; + write_runtime_manifest(paths, &manifest, ManifestOverwrite::Replace)?; let _ = writeln!( output, @@ -1012,7 +1013,7 @@ fn install_tarball_runtime( devel: true, installed_at_unix_ms: unix_time_millis(), }; - save_runtime_manifest(paths, &manifest)?; + write_runtime_manifest(paths, &manifest, ManifestOverwrite::Replace)?; let _ = writeln!(output, " extracted: {}", install_root.display()); let _ = writeln!(output, " manifest: {}", manifest_path.display()); diff --git a/docs/architecture.md b/docs/architecture.md index 1002fb53b5..816e6a6702 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -72,7 +72,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. -- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. +- [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing them (`write_runtime_manifest`, the one routine every way of registering a runtime goes through: an SDK install, `rocm runtimes import`, `rocm runtimes adopt` and the recovery of the runtime `rocm setup` created. It writes the registry entry, and for a runtime ROCm CLI owns (`InstalledRuntimeManifest::owns_install_root`, the same rule uninstall and prune use to decide whether they may delete the folder) the `.rocm-cli-runtime.json` in its install root; an imported or adopted runtime is someone else's folder, so it gets the registry entry only. Its one parameter is `ManifestOverwrite`: an install replaces its own entry, import and adopt refuse an existing one unless `--replace` is given, and recovery refuses so it only fills a missing entry. Neither file is written atomically) and loading them (`load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. diff --git a/docs/testing.md b/docs/testing.md index 97aa511110..406e43f497 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -907,12 +907,31 @@ rocm runtimes adopt \ --runtime-key adopted-release-pip-gfx120x-all-7-13-0 ``` +Neither writes into the folder it registers: the registry entry is the only +file created. The `.rocm-cli-runtime.json` that `rocm install sdk` leaves in an +install root is ROCm CLI's proof that it owns the folder, so an imported or +adopted runtime does not get one, and uninstall and prune never delete such a +folder. A second import or adopt of the same runtime key is refused until +`--replace` is given. `@id:runtime-lifecycle-import-rejects-duplicate-unless-replacing` +(`runtime-lifecycle-06`) checks the import half end to end: the registry entry +exists, the imported folder gains no manifest file, and the duplicate is refused +until `--replace` (the unit tests also check the refused entry is left as it +was). Adopting needs a Python environment to probe and the +recovery of the runtime `rocm setup` created (which restores a missing registry +entry from the manifest already in the setup folder) needs a setup, so neither +has a scenario; the unit tests below assert each path's file set instead, and +`write_runtime_manifest` is the one routine all four paths write through. + Focused unit coverage: ```bash cargo test -p rocm --bin rocm runtime_activation cargo test -p rocm --bin rocm runtime_import cargo test -p rocm --bin rocm runtime_adopt +cargo test -p rocm --bin rocm runtime_install_writes +cargo test -p rocm --bin rocm setup_recovery +cargo test -p rocm --bin rocm write_runtime_manifest +cargo test -p rocm --bin rocm only_the_shared_routine_writes cargo test -p rocm --bin rocm runtime_key_includes_version_for_side_by_side_installs cargo test -p rocm --bin rocm engine_plugin cargo test -p rocm --bin rocm missing_packaged_engine_reason From cf10007bb4455771c890f6dde81a48ff30004ea7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 06:45:28 +0000 Subject: [PATCH 12/20] fix(comfyui): keep a --dry-run install preview from restoring the registry entry With no runtime chosen, `rocm comfyui install` now recovers the runtime `rocm setup` created before counting the ready ones, and recovering writes a lost registry entry back. That ran before the install looked at `--dry-run`, so a preview changed a file even though the flag is documented as "Show what would happen without changing files". The recovery is split into finding the setup runtime (read-only) and writing its entry. `select_single_ready_runtime` takes a `SetupRecovery`: `Persist` writes the entry as before, and `PreviewOnly` counts the setup runtime from its own manifest, normalised and ordered as the registry would return it once restored, and writes nothing, so the preview names the runtime the real run will use. The engine paths keep `Persist`; the install derives the mode from `--dry-run`. The flag help stays true, so no wording changes. comfyui-15 now asserts that the preview selects the runtime and the registry still has no entry, and comfyui-16 runs the install against an existing checkout and asserts the entry is written back. Unit tests pin both modes, that preview and real run list the same ready runtimes in the same order, that an unusable setup runtime is reported alike, and the dry-run mapping. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/install.rs | 106 +++++++- apps/rocm/src/comfyui/runtime_env.rs | 54 ++-- apps/rocm/src/engine_runtime_select.rs | 12 +- apps/rocm/src/runtime_adopt.rs | 239 ++++++++++++++++-- apps/rocm/src/runtime_registry.rs | 2 +- docs/architecture.md | 4 +- docs/testing.md | 13 +- tests/e2e-cucumber/features/comfyui.feature | 27 +- tests/e2e-cucumber/tests/e2e/comfyui_steps.rs | 60 ++++- 9 files changed, 456 insertions(+), 61 deletions(-) diff --git a/apps/rocm/src/comfyui/install.rs b/apps/rocm/src/comfyui/install.rs index 0ac2a164de..6e6b598bbe 100644 --- a/apps/rocm/src/comfyui/install.rs +++ b/apps/rocm/src/comfyui/install.rs @@ -27,6 +27,7 @@ use super::{ }; use crate::cli_progress::AnimatedSpinner; use crate::freeform_planner::format_structured_tool_call; +use crate::runtime_adopt::SetupRecovery; use crate::therock; use anyhow::{Context, Result, bail}; use flate2::read::GzDecoder; @@ -48,7 +49,14 @@ pub(crate) fn install( options: ComfyUiInstallOptions, ) -> Result { paths.ensure()?; - let runtime = select_runtime(paths, config, options.runtime_id.as_deref())?; + // A preview names the runtime the real run will use but does not write the + // registry entry that run would restore. + let runtime = select_runtime( + paths, + config, + options.runtime_id.as_deref(), + SetupRecovery::for_dry_run(options.dry_run), + )?; let app_root = runtime_app_root(&runtime.manifest); let source_path = source_path_from_app_root(&app_root); let log_path = install_log_path_from_app_root(&app_root); @@ -1908,6 +1916,102 @@ mod tests { remove_test_root(&paths); Ok(()) } + + /// `--dry-run` promises not to change files. With nothing chosen, the install + /// finds the runtime `rocm setup` created even though its registry entry was + /// lost, but a preview must not write that entry back: it names the same + /// runtime the real run will use and leaves the registry as it found it. + #[test] + fn dry_run_previews_the_setup_runtime_without_restoring_its_registry_entry() -> Result<()> { + let paths = test_paths("comfyui-dry-run-setup-runtime"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + + let rendered = install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: true, + }, + )?; + + assert!(rendered.contains(" mode: dry-run\n"), "{rendered}"); + assert!( + rendered.contains(&format!( + " ROCm runtime: {}\n", + therock::runtime_version_display(&manifest.version) + )), + "{rendered}" + ); + assert!( + rendered + .lines() + .any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {}", manifest.runtime_key))), + "{rendered}" + ); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "a preview must not restore the registry entry" + ); + remove_test_root(&paths); + Ok(()) + } + + /// The other half: the real run does restore the entry, so the runtime the + /// preview named is one `rocm runtimes list` shows once the install has run. + #[cfg(unix)] + #[test] + fn install_restores_the_setup_runtime_registry_entry_the_preview_left_alone() -> Result<()> { + let paths = test_paths("comfyui-install-setup-runtime"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + let app_root = runtime_app_root(&manifest); + plant_cached_source_archive( + &app_root, + &[("main.py", "new main"), ("requirements.txt", "torch\n")], + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("runtime has no Python")?, + ); + rocm_core::fake_executable::write_fake_executable( + &python, + "#!/bin/sh\nprintf '%s' '{\"torch_version\": \"2.4.0\", \"torch_cuda_available\": true, \"device_count\": 1, \"devices\": [\"Fake GPU\"]}' > \"$2\"\n", + )?; + + install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: false, + }, + )?; + + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + remove_test_root(&paths); + Ok(()) + } } /// Property tests for the ComfyUI source unpack, judged on the filesystem by diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 647296f75a..ee95deafde 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -17,7 +17,7 @@ //! ComfyUI types. use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; -use crate::runtime_adopt::{SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; use anyhow::{Context, Result, bail}; @@ -33,12 +33,13 @@ pub(super) fn select_runtime( paths: &AppPaths, config: &RocmCliConfig, selector: Option<&str>, + recovery: SetupRecovery, ) -> Result { let manifest = match selector.map(str::trim).filter(|value| !value.is_empty()) { Some(selector) => { select_runtime_by_selector(®istered_runtimes(paths)?, selector)?.clone() } - None => select_default_runtime(paths, config)?, + None => select_default_runtime(paths, config, recovery)?, }; if runtime_usability_status(&manifest) != "ready" { bail!( @@ -134,6 +135,7 @@ fn install_rocm_first() -> anyhow::Error { fn select_default_runtime( paths: &AppPaths, config: &RocmCliConfig, + recovery: SetupRecovery, ) -> Result { if let Some(active_key) = config.active_runtime_key.as_deref() { return select_runtime_by_selector(®istered_runtimes(paths)?, active_key).cloned(); @@ -143,7 +145,9 @@ fn select_default_runtime( select_runtime_by_default_id(®istered_runtimes(paths)?, default_runtime_id).cloned() } // Nothing says which runtime to use: the rule the engine paths share. - None => single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config)?), + None => { + single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config, recovery)?) + } } } @@ -425,6 +429,17 @@ mod tests { use rocm_core::{runtime_python_executable_in_env, runtime_rocm_library_filename}; use std::fs; + /// `select_runtime` as a real install runs it: a runtime that only setup + /// recovery can make visible has its registry entry written back. The + /// preview mode has its own tests. + fn persisting_select_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + selector: Option<&str>, + ) -> Result { + select_runtime(paths, config, selector, SetupRecovery::Persist) + } + #[test] fn default_runtime_selection_uses_single_ready_runtime() -> Result<()> { let paths = test_paths("comfyui-single-ready-runtime"); @@ -656,7 +671,7 @@ mod tests { ); // `select_runtime`'s bail when nothing is registered at all. - let error = select_runtime(&paths, &RocmCliConfig::default(), None) + let error = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) .expect_err("no registered runtime means no selection"); let message = error.to_string(); assert!( @@ -863,7 +878,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; write_runtime_manifest(&paths, &unusable)?; - let error = select_runtime( + let error = persisting_select_runtime( &paths, &RocmCliConfig::default(), Some("release-wheel-gfx94x-dcgpu-7-13-0"), @@ -981,7 +996,7 @@ mod tests { -> Result<()> { let paths = test_paths("comfyui-selection-none-registered"); - let message = select_runtime(&paths, &RocmCliConfig::default(), None) + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) .expect_err("no registered runtime means no selection") .to_string(); @@ -1006,7 +1021,7 @@ mod tests { 20, )?; - let selected = select_runtime(&paths, &RocmCliConfig::default(), None)?; + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); assert_eq!( @@ -1037,7 +1052,7 @@ mod tests { ); fs::remove_file(&python)?; - let message = select_runtime(&paths, &RocmCliConfig::default(), None) + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) .expect_err("an unusable runtime is not ready") .to_string(); @@ -1053,7 +1068,7 @@ mod tests { ); fs::write(&python, "python")?; - let selected = select_runtime(&paths, &RocmCliConfig::default(), None)?; + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); Ok(()) } @@ -1080,7 +1095,7 @@ mod tests { )?; let mut config = RocmCliConfig::default(); - let message = select_runtime(&paths, &config, None) + let message = persisting_select_runtime(&paths, &config, None) .expect_err("several ready runtimes and no choice must not select") .to_string(); @@ -1093,11 +1108,11 @@ mod tests { "unexpected refusal: {message}" ); - let selected = select_runtime(&paths, &config, Some(&first.runtime_key))?; + let selected = persisting_select_runtime(&paths, &config, Some(&first.runtime_key))?; assert_eq!(selected.manifest.runtime_key, first.runtime_key); crate::runtimes_cmd::activate_runtime(&paths, &mut config, &second.runtime_key)?; - let selected = select_runtime(&paths, &config, None)?; + let selected = persisting_select_runtime(&paths, &config, None)?; assert_eq!(selected.manifest.runtime_key, second.runtime_key); Ok(()) } @@ -1120,7 +1135,7 @@ mod tests { ); assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); - let selected = select_runtime(&paths, &config, None)?; + let selected = persisting_select_runtime(&paths, &config, None)?; assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); assert_eq!( @@ -1149,7 +1164,7 @@ mod tests { )?; let config = RocmCliConfig::load(&paths)?; - let message = select_runtime(&paths, &config, None) + let message = persisting_select_runtime(&paths, &config, None) .expect_err("an unusable setup runtime is an error") .to_string(); @@ -1252,10 +1267,13 @@ mod tests { }; let comfyui_paths = test_paths(&format!("comfyui-engine-agree-{label}-comfyui")); plant_state(&comfyui_paths)?; - let comfyui = - select_runtime(&comfyui_paths, &RocmCliConfig::load(&comfyui_paths)?, None) - .ok() - .map(|selected| selected.manifest.runtime_key); + let comfyui = persisting_select_runtime( + &comfyui_paths, + &RocmCliConfig::load(&comfyui_paths)?, + None, + ) + .ok() + .map(|selected| selected.manifest.runtime_key); let engine_paths = test_paths(&format!("comfyui-engine-agree-{label}-engine")); plant_state(&engine_paths)?; let engine = validate_engine_selection_runtime( diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 46a1cf8866..061f968b49 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -106,7 +106,7 @@ use rocm_core::{ }; use rocm_engine_protocol::EngineRecipeHint; -use crate::runtime_adopt::{SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; use crate::runtimes_cmd::{recover_setup_runtime_registration, select_runtime_manifest}; use crate::therock; @@ -214,10 +214,12 @@ fn single_ready_runtime_key(paths: &AppPaths) -> Result> { let config = RocmCliConfig::load(paths).unwrap_or_default(); // Zero or several ready runtimes leave the selection open: the caller that // needs a runtime says so in its own words. - Ok(match select_single_ready_runtime(paths, &config)? { - SingleReadyRuntime::One(manifest) => Some(manifest.runtime_key), - SingleReadyRuntime::Zero { .. } | SingleReadyRuntime::Many { .. } => None, - }) + Ok( + match select_single_ready_runtime(paths, &config, SetupRecovery::Persist)? { + SingleReadyRuntime::One(manifest) => Some(manifest.runtime_key), + SingleReadyRuntime::Zero { .. } | SingleReadyRuntime::Many { .. } => None, + }, + ) } pub(crate) fn resolve_runtime_selector_to_exact_key( diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 60862770dc..6c080accf5 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -27,7 +27,7 @@ use rocm_core::{AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_e use crate::runtime_registry::{ paths_equivalent, runtime_manifest_path, runtime_registry_dir, - validate_runtime_manifest_for_activation, + sort_manifests_newest_install_first, validate_runtime_manifest_for_activation, }; use crate::therock; @@ -381,10 +381,17 @@ fn write_runtime_registry_manifest( Ok(()) } -pub(crate) fn recover_setup_runtime_registration( - paths: &AppPaths, - config: &RocmCliConfig, -) -> Result> { +/// The runtime `rocm setup` created, as the manifest in its install tree +/// describes it, and whether the runtime registry already holds its entry. +struct SetupRuntime { + manifest: therock::InstalledRuntimeManifest, + registered: bool, +} + +/// Find the runtime `rocm setup` created and check that it is usable, without +/// changing anything. `None` when setup named no folder or the folder holds no +/// manifest; an error when it does and the runtime cannot be used. +fn find_setup_runtime(paths: &AppPaths, config: &RocmCliConfig) -> Result> { let Some(setup_root) = config .setup .therock_venv @@ -428,16 +435,53 @@ pub(crate) fn recover_setup_runtime_registration( ) })?; - if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { - write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { + let registered = runtime_manifest_path(paths, &manifest.runtime_key).is_file(); + Ok(Some(SetupRuntime { + manifest, + registered, + })) +} + +pub(crate) fn recover_setup_runtime_registration( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result> { + let Some(setup) = find_setup_runtime(paths, config)? else { + return Ok(None); + }; + if !setup.registered { + write_runtime_registry_manifest(paths, &setup.manifest, false).with_context(|| { format!( "failed to restore setup runtime `{}` into {}", - manifest.runtime_key, + setup.manifest.runtime_key, runtime_registry_dir(paths).display() ) })?; } - Ok(Some(manifest.runtime_key)) + Ok(Some(setup.manifest.runtime_key)) +} + +/// Whether finding the runtime `rocm setup` created may write its lost registry +/// entry back while a runtime is being picked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SetupRecovery { + /// Write the entry back, as `rocm serve` and `rocm engines install` do. + Persist, + /// Count the setup runtime as if its entry were there, and write nothing: + /// a `--dry-run` preview must name the runtime the real run will use + /// without changing any file. + PreviewOnly, +} + +impl SetupRecovery { + /// The recovery a command with a `--dry-run` flag uses. + pub(crate) const fn for_dry_run(dry_run: bool) -> Self { + if dry_run { + Self::PreviewOnly + } else { + Self::Persist + } + } } /// What picking "the one ready runtime" found. The caller decides how to report @@ -467,16 +511,42 @@ pub(crate) enum SingleReadyRuntime { /// Ready means `validate_runtime_manifest_for_activation` accepts the runtime, /// the same check `rocm runtimes activate` applies. Setup is recovered first /// (`recover_setup_runtime_registration`), so the runtime `rocm setup` created -/// counts even if its registry entry was lost, and recovering it writes that -/// entry back. +/// counts even if its registry entry was lost. With [`SetupRecovery::Persist`] +/// that writes the entry back; with [`SetupRecovery::PreviewOnly`] nothing is +/// written, and the setup runtime is counted from its own manifest as the +/// registry would hold it once restored, so the answer is the one `Persist` +/// gives. pub(crate) fn select_single_ready_runtime( paths: &AppPaths, config: &RocmCliConfig, + recovery: SetupRecovery, ) -> Result { - recover_setup_runtime_registration(paths, config)?; - Ok(classify_single_ready_runtime( - therock::load_runtime_manifests(paths)?, - )) + match recovery { + SetupRecovery::Persist => { + recover_setup_runtime_registration(paths, config)?; + Ok(classify_single_ready_runtime( + therock::load_runtime_manifests(paths)?, + )) + } + SetupRecovery::PreviewOnly => { + let setup = find_setup_runtime(paths, config)?; + let mut registered = therock::load_runtime_manifests(paths)?; + if let Some(setup) = setup + && !setup.registered + { + // What `load_runtime_manifests` would return for the entry that + // `Persist` would have written. + registered.push( + setup + .manifest + .normalize_storage_paths() + .normalize_host_paths(), + ); + sort_manifests_newest_install_first(&mut registered); + } + Ok(classify_single_ready_runtime(registered)) + } + } } /// The pure half of [`select_single_ready_runtime`]: which of `registered` are @@ -941,7 +1011,7 @@ mod tests { let config = RocmCliConfig::load(&paths)?; assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); - match select_single_ready_runtime(&paths, &config)? { + match select_single_ready_runtime(&paths, &config, SetupRecovery::Persist)? { SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-s"), other => panic!("expected the recovered setup runtime, got {other:?}"), } @@ -953,7 +1023,11 @@ mod tests { // Without setup naming a folder there is nothing to recover and the // registry is not touched. let (other_root, other_paths) = test_paths("single-ready-no-setup"); - match select_single_ready_runtime(&other_paths, &RocmCliConfig::default())? { + match select_single_ready_runtime( + &other_paths, + &RocmCliConfig::default(), + SetupRecovery::Persist, + )? { SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), other => panic!("expected no ready runtime, got {other:?}"), } @@ -962,4 +1036,135 @@ mod tests { let _ = fs::remove_dir_all(other_root); Ok(()) } + + /// The keys `select_single_ready_runtime` found ready, in the order it lists + /// them, or the message of its refusal to recover. + fn single_ready_outcome( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, + ) -> Result, String> { + match select_single_ready_runtime(paths, config, recovery) { + Ok(SingleReadyRuntime::One(manifest)) => Ok(vec![manifest.runtime_key]), + Ok(SingleReadyRuntime::Many { ready }) => { + Ok(ready.into_iter().map(|m| m.runtime_key).collect()) + } + Ok(SingleReadyRuntime::Zero { registered }) => { + Err(format!("none ready of {}", registered.len())) + } + Err(error) => Err(format!("{error:#}")), + } + } + + /// A preview counts the runtime setup created from its own manifest and + /// writes nothing; the real run then finds the same runtime and writes the + /// entry. Both orders of install time are covered: the ready runtimes are + /// listed newest first, and a preview must list them the way the registry + /// will once the entry is back. + #[test] + fn preview_counts_the_setup_runtime_without_writing_and_agrees_with_persist() -> Result<()> { + for (label, other_installed_at) in [("other-newer", 30), ("other-older", 10)] { + let (root, paths) = test_paths(&format!("single-ready-preview-{label}")); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-setup", + )?; + write_test_pip_runtime( + &paths, + "runtime-other", + "therock-release:gfx120X-all", + "7.14.0", + other_installed_at, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry_before = crate::test_support::runtime_registry_entry_names(&paths); + assert_eq!(registry_before, ["runtime-other.json"], "{label}"); + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect("the setup runtime and the other one are both ready"); + + assert_eq!(preview.len(), 2, "{label}: {preview:?}"); + assert!(preview.contains(&lost.runtime_key), "{label}: {preview:?}"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + registry_before, + "{label}: a preview must not write the lost entry" + ); + let persisted = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect("recovery finds the same two"); + assert_eq!(preview, persisted, "{label}: preview and real run disagree"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + [ + "runtime-other.json".to_owned(), + "runtime-setup.json".to_owned() + ], + "{label}: the real run writes the entry back" + ); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + /// The setup runtime alone is "the one ready runtime" in a preview, and a + /// registry that already holds it is left as it is. + #[test] + fn preview_picks_a_lost_setup_runtime_and_ignores_a_registered_one() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-one"); + crate::test_support::write_test_setup_runtime_missing_from_registry(&paths, "runtime-s")?; + let config = RocmCliConfig::load(&paths)?; + + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + // Once the entry exists a preview and a real run see one runtime, and + // neither adds a second copy of it. + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::Persist), + Ok(vec!["runtime-s".to_owned()]) + ); + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A setup runtime that is no longer usable is reported by a preview as the + /// real run reports it, and neither writes anything. + #[test] + fn preview_reports_an_unusable_setup_runtime_as_persist_does() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-unusable"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + fs::remove_file( + lost.python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect_err("an unusable setup runtime is an error"); + let persist = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect_err("an unusable setup runtime is an error"); + + assert!(preview.contains("is not usable"), "{preview}"); + assert_eq!(preview, persist); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn a_dry_run_previews_and_a_real_run_persists() { + assert_eq!(SetupRecovery::for_dry_run(true), SetupRecovery::PreviewOnly); + assert_eq!(SetupRecovery::for_dry_run(false), SetupRecovery::Persist); + } } diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 9f9fa17ce7..5e0ce19d02 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -121,7 +121,7 @@ impl InstalledRuntimeManifest { wheel_composition_includes_devel(self.wheel_composition.as_ref()).unwrap_or(self.devel) } - fn normalize_host_paths(mut self) -> Self { + pub(crate) fn normalize_host_paths(mut self) -> Self { self.install_root = normalize_manifest_path(self.install_root); self.python_launcher = self .python_launcher diff --git a/docs/architecture.md b/docs/architecture.md index c87df423da..4ec3b98f78 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. - [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. - [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) — the argv helpers that `chat.rs`, `tool_policy.rs` and the freeform planner share, in a leaf module whose only `crate::` dependency is the leaf [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs): `chat_cli_arg_value`, `chat_cli_arg_value_checked`, `chat_cli_has_flag` and `ensure_flag` (from `chat.rs`), and `format_structured_tool_call` (which quotes each argument with `shell_quote`), `serve_args_request_cpu_device` and `device_policy_is_cpu_mode` (from `freeform_planner.rs`). They moved so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `ensure_flag` (read by `freeform_execution.rs`) and `freeform_planner.rs` re-exports `format_structured_tool_call`, so every other caller keeps its path. -- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. +- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Its `SetupRecovery` argument says whether recovering the setup runtime may write its lost registry entry back (`Persist`, what the engine paths and a real ComfyUI install use) or only count the runtime from its own manifest and write nothing (`PreviewOnly`, for a `--dry-run` preview, which must name the runtime the real run will use without changing a file); `SetupRecovery::for_dry_run` maps the flag. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. @@ -115,7 +115,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: - [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). - [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. - [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. +- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. - [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. - [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. diff --git a/docs/testing.md b/docs/testing.md index ddeb858f5d..f30b66650f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -780,13 +780,16 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. -`comfyui-15` covers which runtime `rocm comfyui install` uses when none is -chosen: it recovers the runtime `rocm setup` created before counting the ready -ones, as `rocm serve` does, so a runtime whose registry entry was lost is found -and its entry written back. It runs with no GPU, through the install's dry run: +`comfyui-15` and `comfyui-16` cover which runtime `rocm comfyui install` uses +when none is chosen: it recovers the runtime `rocm setup` created before +counting the ready ones, as `rocm serve` does, so a runtime whose registry entry +was lost is found. The real run (`comfyui-16`, against an existing checkout that +needs no dependencies) writes the entry back; the `--dry-run` preview +(`comfyui-15`) names the same runtime but writes nothing, because a preview +promises not to change files. Both run with no GPU: ```bash -cargo xtask e2e -- -n "comfyui-15" +cargo xtask e2e -- -n "comfyui-1[56]" ``` `comfyui-14` is different: it installs ComfyUI into a real, isolated managed diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index 1413cadefb..9aee470019 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -333,14 +333,29 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv # and it finds it the way `rocm serve` and `rocm engines install` do: it # recovers the runtime `rocm setup` created first. A runtime whose registry # entry was lost, but whose install tree still holds its own manifest and that - # setup names, therefore counts as the ready runtime and its entry is written - # back, instead of the install refusing with "Install ROCm first" while setup - # has finished. The dry run is the cheapest way to see which runtime the - # install chose: it prints the `--runtime-id` of the command it would run. - # No GPU is needed: readiness is filesystem and manifest state. + # setup names, therefore counts as the ready runtime, instead of the install + # refusing with "Install ROCm first" while setup has finished. The real run + # writes the lost registry entry back (comfyui-16); a `--dry-run` preview + # names the same runtime but leaves the registry as it found it, because it + # promises not to change files (this scenario). The preview prints the + # `--runtime-id` of the command it would run, which is how it shows the + # choice. No GPU is needed: readiness is filesystem and manifest state. @id:comfyui-install-finds-the-runtime-setup-created - Scenario: comfyui-15 - ComfyUI install finds the runtime setup created when the registry lost its entry + Scenario: comfyui-15 - A ComfyUI install preview finds the runtime setup created without writing the registry Given a ready ROCm runtime that setup created and the registry has forgotten When the user previews a ComfyUI install without choosing a runtime Then the preview selects that runtime + And the runtime registry still has no entry for it + + # The real run of the same install: it uses the runtime setup created and + # writes its lost registry entry back, so the runtime it installed into is one + # `rocm runtimes list` shows afterwards. The ComfyUI checkout is already in + # place and asks for nothing beyond the torch stack the runtime provides, so + # the install needs no network and no package index. + @id:comfyui-install-restores-the-runtime-setup-created + Scenario: comfyui-16 - A ComfyUI install restores the registry entry of the runtime setup created + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + When the user installs ComfyUI without choosing a runtime + Then the install uses the runtime setup created And the runtime registry has an entry for it again diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index edf3cba9cc..98f75637aa 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,12 +34,14 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! -//! `comfyui-15` covers the other half of that selection: with nothing chosen, -//! the install recovers the runtime `rocm setup` created before it counts the -//! ready ones, as `rocm serve` does, so a runtime the registry lost is found and -//! its entry written back. It plants a ready runtime and the setup folder that -//! names it, but no registry entry, and previews the install with `--dry-run`, -//! which prints the `--runtime-id` of the runtime it chose. +//! `comfyui-15` and `comfyui-16` cover the other half of that selection: with +//! nothing chosen, the install recovers the runtime `rocm setup` created before +//! it counts the ready ones, as `rocm serve` does, so a runtime the registry +//! lost is found. They plant a ready runtime and the setup folder that names it, +//! but no registry entry. `comfyui-15` previews the install with `--dry-run`, +//! which prints the `--runtime-id` of the runtime it chose and writes nothing, +//! so the registry still has no entry afterwards; `comfyui-16` runs the install +//! against an existing checkout and finds the entry written back. //! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for @@ -623,6 +625,52 @@ async fn registry_has_the_setup_runtime_again(world: &mut E2eWorld) { ); } +/// A preview promises not to change files, and the registry entry is a file: the +/// runtime it named is not registered by having been previewed. +#[then("the runtime registry still has no entry for it")] +async fn registry_still_lacks_the_setup_runtime(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + !entry.exists(), + "a `--dry-run` preview must not write {}", + entry.display() + ); +} + +#[given("a ComfyUI checkout that needs nothing beyond the runtime's torch stack")] +async fn comfyui_checkout_needing_no_dependencies(world: &mut E2eWorld) { + let source_dir = data_dir(world) + .join("runtimes") + .join("roots") + .join(SETUP_RUNTIME_KEY) + .join("apps") + .join("comfyui") + .join("source"); + // Every ComfyUI release has a `main.py`. The requirements name only the + // torch stack the runtime provides, which the install filters out, so it + // reaches no package index. + write_fixture(&source_dir.join("main.py"), ""); + write_fixture( + &source_dir.join("requirements.txt"), + "torch==2.4.0\ntorchvision==0.19.0\ntorchaudio==2.4.0\n", + ); +} + +#[then("the install uses the runtime setup created")] +async fn install_uses_the_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the install should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.contains("action: install") && stdout.contains("ROCm runtime: 7.13.0"), + "the install should report the runtime setup created, got:\n{text}" + ); +} + /// Pacing knobs for `comfyui-04`'s download server, mirroring /// `therock_steps.rs`'s `PACED_TARBALL_*` constants: large enough that several /// chunk boundaries land before the transfer completes, slow enough per chunk From 049aa06ad3a9f8aa4c62cdc9143d7467b1f128a3 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 07:03:22 +0000 Subject: [PATCH 13/20] fix(runtimes): judge an install root manifest as current by what it decodes to The shared manifest writer skips rewriting the `.rocm-cli-runtime.json` in an install root when it already holds what would be written, so recovering the runtime `rocm setup` created does not fail on a folder the user cannot write to. It compared bytes. A manifest an older binary wrote omits keys that now read back as defaults (`wheel_composition` and other optional fields) and serialize as explicit values, so the bytes differ although the manifest is the same. Recovery then wrote the missing registry entry and failed rewriting the otherwise usable, read-only file. The writer now also treats a file as current when it decodes to the manifest that would be written once normalised for storage (`install_root_manifest_is_current`). A different manifest, or a file that does not decode, is still rewritten. The regression test recovers a setup runtime whose read-only install root manifest is in the older shape and asserts the registry entry is restored and the file's bytes are untouched; a pure test covers the equal, same-manifest, different-manifest and not-a-manifest cases on every platform. Signed-off-by: Jussi Elo --- apps/rocm/src/runtime_registry.rs | 76 +++++++++++++++++++++++++++++-- apps/rocm/src/runtimes_cmd.rs | 71 +++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 4 deletions(-) diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 5477674dde..fd75c7707d 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -381,8 +381,12 @@ impl ManifestOverwrite { /// gets none: a root the user may not be able to write to must not make /// registering it fail, and an import's source manifest may be that very file /// in another install. -/// The file is not rewritten when it already holds exactly what would be -/// written, so recovering a runtime from its own manifest leaves it alone. +/// The file is not rewritten when it already holds what would be written, so +/// recovering a runtime from its own manifest leaves it alone. "Holds" is judged +/// on the decoded manifest, not the bytes ([`install_root_manifest_is_current`]): +/// a file an older binary wrote omits keys that now read back as defaults and +/// are written out explicitly, and a root the user cannot write to must not turn +/// that into a failure. /// /// Neither file is written atomically; that is a separate piece of work shared /// with the other writers in the workspace. @@ -411,8 +415,8 @@ pub(crate) fn write_runtime_manifest( if manifest.owns_install_root() { let local_manifest_path = manifest.install_root.join(".rocm-cli-runtime.json"); - let already_current = - fs::read(&local_manifest_path).is_ok_and(|existing| existing == encoded); + let already_current = fs::read(&local_manifest_path) + .is_ok_and(|existing| install_root_manifest_is_current(&existing, &encoded)); if !already_current { fs::write(&local_manifest_path, &encoded) .with_context(|| format!("failed to write {}", local_manifest_path.display()))?; @@ -421,6 +425,22 @@ pub(crate) fn write_runtime_manifest( Ok(()) } +/// Whether an install root manifest already holds the manifest that `encoded` +/// is the storage form of. Equal bytes qualify, and so does a file that decodes +/// to the same manifest once normalised for storage: a manifest written before +/// a field existed has no key for it, reads back as the field's default, and +/// serializes with an explicit value, so a byte comparison reads it as changed. +/// A file that does not decode is not current. +fn install_root_manifest_is_current(existing: &[u8], encoded: &[u8]) -> bool { + if existing == encoded { + return true; + } + serde_json::from_slice::(existing) + .ok() + .and_then(|decoded| serde_json::to_vec_pretty(&decoded.normalize_storage_paths()).ok()) + .is_some_and(|normalised| normalised == encoded) +} + pub(crate) fn load_runtime_manifests(paths: &AppPaths) -> Result> { Ok(load_runtime_manifests_reporting_unparsed(paths)?.0) } @@ -1040,6 +1060,54 @@ mod tests { Ok(()) } + /// The state matrix for "is the install root manifest already current": + /// the same bytes, the same manifest in an older shape (keys that hold no + /// value omitted, so a byte comparison reads it as changed), a different + /// manifest in that shape, and a file that is not a manifest at all. Pure, so + /// it runs on every platform, unlike the read-only-file tests that show the + /// effect on the filesystem. + #[test] + fn install_root_manifest_is_current_compares_the_decoded_manifest() -> Result<()> { + let (root, _paths) = test_paths("manifest-current-decoded"); + let manifest = manifest_in(&root, "release-wheel-gfx120X-all-7.13.0", false, None)?; + let stored = manifest.clone().normalize_storage_paths(); + let encoded = serde_json::to_vec_pretty(&stored)?; + let legacy_shaped = |manifest: &InstalledRuntimeManifest| -> Result> { + let mut document = serde_json::to_value(manifest.clone().normalize_storage_paths())?; + let object = document.as_object_mut().context("a JSON object")?; + let empty: Vec = object + .iter() + .filter(|(_, value)| value.is_null()) + .map(|(key, _)| key.clone()) + .collect(); + assert!(!empty.is_empty(), "the fixture needs a null-valued key"); + for key in empty { + object.remove(&key); + } + Ok(serde_json::to_vec_pretty(&document)?) + }; + + assert!(install_root_manifest_is_current(&encoded, &encoded)); + let legacy = legacy_shaped(&manifest)?; + assert_ne!(legacy, encoded, "the legacy shape must differ in bytes"); + assert!( + install_root_manifest_is_current(&legacy, &encoded), + "a legacy-shaped file that decodes to the same manifest is current" + ); + let newer = InstalledRuntimeManifest { + version: "7.14.0".to_owned(), + ..manifest + }; + assert!( + !install_root_manifest_is_current(&legacy_shaped(&newer)?, &encoded), + "a legacy-shaped file that decodes to a different manifest is not current" + ); + assert!(!install_root_manifest_is_current(b"not json", &encoded)); + assert!(!install_root_manifest_is_current(b"", &encoded)); + let _ = fs::remove_dir_all(&root); + Ok(()) + } + /// Every way of registering a runtime goes through /// [`write_runtime_manifest`]. A second writer would be free to drift again /// (a different overwrite rule, a missing install-root manifest), so no diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 8301cb2c8b..885e0bade4 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -2613,6 +2613,77 @@ pub(crate) mod tests { Ok(()) } + /// Rewrite a manifest file as an older rocm-cli wrote it: without the keys + /// that hold no value, which a newer binary reads back as their defaults and + /// writes out as explicit `null`s. Returns the bytes now on disk. + fn rewrite_as_legacy_shaped(path: &Path) -> Result> { + let mut document: serde_json::Value = serde_json::from_slice(&fs::read(path)?)?; + let object = document + .as_object_mut() + .context("a manifest is a JSON object")?; + let empty: Vec = object + .iter() + .filter(|(_, value)| value.is_null()) + .map(|(key, _)| key.clone()) + .collect(); + assert!( + !empty.is_empty(), + "the fixture must hold a null-valued key for this to mean anything" + ); + for key in empty { + object.remove(&key); + } + let legacy = serde_json::to_vec_pretty(&document)?; + fs::write(path, &legacy)?; + Ok(legacy) + } + + /// Recovering the setup runtime from a manifest an older binary wrote, in a + /// folder the user cannot write to: the file decodes to the very manifest the + /// recovery would write, so it is already current and is left alone. Judging + /// it by bytes would read the missing keys as a change, write the registry + /// entry, and then fail on the read-only file. The registry entry and the + /// file's bytes are asserted together. (Where the file stays writable + /// regardless, as for an administrator, the unchanged bytes still prove the + /// file was not rewritten.) + #[test] + #[allow(clippy::permissions_set_readonly_false)] + fn setup_recovery_leaves_a_read_only_legacy_shaped_install_root_manifest_alone() -> Result<()> { + let (root, paths) = test_paths("runtime-files-recovery-legacy"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + let entry = runtime_manifest_path(&paths, &manifest.runtime_key); + let local = manifest.install_root.join(".rocm-cli-runtime.json"); + fs::remove_file(&entry)?; + let legacy = rewrite_as_legacy_shaped(&local)?; + let mut permissions = fs::metadata(&local)?.permissions(); + permissions.set_readonly(true); + fs::set_permissions(&local, permissions.clone())?; + + let recovered = recover_setup_runtime_registration(&paths, &config); + + permissions.set_readonly(false); + fs::set_permissions(&local, permissions)?; + let recovered = recovered + .context("a usable manifest in a folder that cannot be written is recovered")?; + assert_eq!(recovered.as_deref(), Some(manifest.runtime_key.as_str())); + assert!(entry.is_file(), "the registry entry is restored"); + assert_eq!( + fs::read(&local)?, + legacy, + "the install root manifest is already current and must not be rewritten" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + /// The writer gives a runtime a manifest in its folder exactly when /// uninstall, which prune asks the same question of, may delete that folder. /// The two are written in different modules; this is what ties them. From 02506ce40700b09d2efcea7e8fd93fcaf5b57f80 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 08:32:13 +0000 Subject: [PATCH 14/20] fix(comfyui): let an explicit runtime choice find the setup runtime a preview named `rocm comfyui install --dry-run` finds the runtime `rocm setup` created even when its registry entry was lost, and prints `rocm comfyui install --runtime-id ` for it without writing the entry. That printed command then failed from the state the preview left: an explicit choice only looked in the registry, so it refused with "Install ROCm first" or "runtime not found". A choice that names a runtime (`--runtime-id`, the active runtime, the configured default) is still answered by the registry first. When the registry alone cannot answer, it is looked up once more among the registry plus the setup runtime, the way the engine paths resolve an explicit `--runtime-id`. `SetupRecovery` decides the write: a preview adds the runtime to the list and writes nothing, a real run writes the entry back. If the setup runtime cannot be used, the registry's own refusal stands rather than an error about a runtime the user did not name. The list the single-ready rule counts from is now `registered_runtimes_with_setup`, shared with the new lookup. Covered by unit tests that follow the printed command from the unchanged registry (preview again, then a real run), a matrix of choices by recovery mode, and the choices that must not change; and by scenario comfyui-17, which runs the printed command as the CLI prints it. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/install.rs | 78 ++++++- apps/rocm/src/comfyui/runtime_env.rs | 195 ++++++++++++++++-- apps/rocm/src/runtime_adopt.rs | 23 ++- docs/architecture.md | 4 +- docs/testing.md | 10 +- tests/e2e-cucumber/features/comfyui.feature | 19 +- tests/e2e-cucumber/tests/e2e/comfyui_steps.rs | 34 ++- 7 files changed, 331 insertions(+), 32 deletions(-) diff --git a/apps/rocm/src/comfyui/install.rs b/apps/rocm/src/comfyui/install.rs index 6e6b598bbe..c446ddc3b6 100644 --- a/apps/rocm/src/comfyui/install.rs +++ b/apps/rocm/src/comfyui/install.rs @@ -1968,12 +1968,14 @@ mod tests { Ok(()) } - /// The other half: the real run does restore the entry, so the runtime the - /// preview named is one `rocm runtimes list` shows once the install has run. + /// A runtime that setup created and the registry forgot, with a cached + /// ComfyUI archive and a fake interpreter, so a real install into it needs + /// no network and no GPU. #[cfg(unix)] - #[test] - fn install_restores_the_setup_runtime_registry_entry_the_preview_left_alone() -> Result<()> { - let paths = test_paths("comfyui-install-setup-runtime"); + fn setup_runtime_the_registry_forgot( + name: &str, + ) -> Result<(AppPaths, RocmCliConfig, therock::InstalledRuntimeManifest)> { + let paths = test_paths(name); let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( &paths, "release-pip-gfx120x-all-7-14-0", @@ -1994,6 +1996,72 @@ mod tests { &python, "#!/bin/sh\nprintf '%s' '{\"torch_version\": \"2.4.0\", \"torch_cuda_available\": true, \"device_count\": 1, \"devices\": [\"Fake GPU\"]}' > \"$2\"\n", )?; + Ok((paths, config, manifest)) + } + + /// The preview prints the install command it would run, and that command + /// has to work from the state the preview left: the registry still without + /// the entry. Its `--runtime-id` is an explicit choice, which used to look + /// only in the registry and fail with "Install ROCm first". Following the + /// printed command previews again without a write, and runs for real with + /// the one write the real run makes. + #[cfg(unix)] + #[test] + fn the_printed_install_command_works_from_the_state_the_preview_left() -> Result<()> { + let (paths, config, manifest) = + setup_runtime_the_registry_forgot("comfyui-printed-command-setup-runtime")?; + let options = |runtime_id: Option, dry_run| ComfyUiInstallOptions { + runtime_id, + reinstall: false, + dry_run, + }; + let printed_runtime_id = |rendered: &str| -> Result { + let line = rendered + .lines() + .find(|line| line.contains("install command:")) + .with_context(|| format!("the preview prints its command:\n{rendered}"))?; + let words: Vec<&str> = line.split_whitespace().collect(); + let at = words + .iter() + .position(|word| *word == "--runtime-id") + .context("the printed command names the runtime")?; + Ok(words[at + 1].to_owned()) + }; + + let preview = install(&paths, &config, options(None, true))?; + let printed = printed_runtime_id(&preview)?; + assert_eq!(printed, manifest.runtime_key, "{preview}"); + assert!( + crate::test_support::runtime_registry_entry_names(&paths).is_empty(), + "the premise: the preview leaves the registry without the entry" + ); + + let again = install(&paths, &config, options(Some(printed.clone()), true)) + .context("the printed command, previewed again")?; + assert_eq!(printed_runtime_id(&again)?, printed, "{again}"); + assert!( + crate::test_support::runtime_registry_entry_names(&paths).is_empty(), + "previewing the printed command must not write the entry either" + ); + + install(&paths, &config, options(Some(printed), false)) + .context("the printed command, run for real")?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)], + "the real run restores the entry the printed command's runtime needs" + ); + remove_test_root(&paths); + Ok(()) + } + + /// The other half: the real run does restore the entry, so the runtime the + /// preview named is one `rocm runtimes list` shows once the install has run. + #[cfg(unix)] + #[test] + fn install_restores_the_setup_runtime_registry_entry_the_preview_left_alone() -> Result<()> { + let (paths, config, manifest) = + setup_runtime_the_registry_forgot("comfyui-install-setup-runtime")?; install( &paths, diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index cbbe106ba4..24663f3975 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -7,7 +7,10 @@ //! //! [`select_runtime`] turns an explicit `--runtime-id` or the configured //! default into the one ready managed Python runtime an install runs against, -//! and explains how to pick one when none or several qualify. The +//! and explains how to pick one when none or several qualify. A choice that +//! names the runtime `rocm setup` created finds it even when its registry entry +//! was lost (see `pick_from_registry`), so the `--runtime-id` an install preview +//! prints works from the state the preview left. The //! `runtime_environment_*` functions build the `VIRTUAL_ENV`, `ROCM_PATH`, //! `PATH` and library-path settings a ComfyUI child process runs with, and //! [`apply_runtime_environment`] puts them on a command. @@ -17,7 +20,9 @@ //! ComfyUI types. use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; -use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtime_adopt::{ + SetupRecovery, SingleReadyRuntime, registered_runtimes_with_setup, select_single_ready_runtime, +}; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; use anyhow::{Context, Result, bail}; @@ -36,9 +41,9 @@ pub(super) fn select_runtime( recovery: SetupRecovery, ) -> Result { let manifest = match selector.map(str::trim).filter(|value| !value.is_empty()) { - Some(selector) => { - select_runtime_by_selector(®istered_runtimes(paths)?, selector)?.clone() - } + Some(selector) => pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_selector(manifests, selector) + })?, None => select_default_runtime(paths, config, recovery)?, }; if runtime_usability_status(&manifest) != "ready" { @@ -117,13 +122,36 @@ fn format_runtime_statuses<'a>( .join(", ") } -/// Every registered runtime, or the refusal to install ROCm when there is none. -fn registered_runtimes(paths: &AppPaths) -> Result> { - let manifests = therock::load_runtime_manifests(paths)?; - if manifests.is_empty() { - return Err(install_rocm_first()); +/// Apply `pick` (a choice that names a runtime: `--runtime-id`, the active +/// runtime or the configured default) to the registry. A runtime that `rocm +/// setup` created and whose registry entry was lost is not a refusal either: +/// when the registry alone has no answer, `pick` runs once more over the +/// registry plus that runtime, the way the engine paths resolve an explicit +/// `--runtime-id`. Which write that implies is `recovery`'s: a preview adds the +/// runtime to the list without writing its entry, so the `--runtime-id` an +/// install preview prints is a command that works from the state the preview +/// left. If setup's runtime cannot be used, the registry's own answer stands. +fn pick_from_registry( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, + pick: impl for<'a> Fn( + &'a [therock::InstalledRuntimeManifest], + ) -> Result<&'a therock::InstalledRuntimeManifest>, +) -> Result { + let registered = therock::load_runtime_manifests(paths)?; + let refusal = if registered.is_empty() { + install_rocm_first() + } else { + match pick(®istered) { + Ok(manifest) => return Ok(manifest.clone()), + Err(error) => error, + } + }; + match registered_runtimes_with_setup(paths, config, recovery) { + Ok(with_setup) if with_setup.len() > registered.len() => pick(&with_setup).cloned(), + _ => Err(refusal), } - Ok(manifests) } fn install_rocm_first() -> anyhow::Error { @@ -138,12 +166,14 @@ fn select_default_runtime( recovery: SetupRecovery, ) -> Result { if let Some(active_key) = config.active_runtime_key.as_deref() { - return select_runtime_by_selector(®istered_runtimes(paths)?, active_key).cloned(); + return pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_selector(manifests, active_key) + }); } match config.default_runtime_id.as_deref() { - Some(default_runtime_id) => { - select_runtime_by_default_id(®istered_runtimes(paths)?, default_runtime_id).cloned() - } + Some(default_runtime_id) => pick_from_registry(paths, config, recovery, |manifests| { + select_runtime_by_default_id(manifests, default_runtime_id) + }), // Nothing says which runtime to use: the rule the engine paths share. None => { single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config, recovery)?) @@ -1167,6 +1197,141 @@ mod tests { Ok(()) } + /// A choice that names the runtime setup created, by `--runtime-id` (key or + /// id), the active runtime or the configured default, finds it even though + /// its registry entry was lost. That is what makes the `--runtime-id` an + /// install preview prints a command that works from the state the preview + /// left: a preview adds the runtime to what it searches and writes nothing, + /// a real run writes the entry back. Each row asserts the pick and the + /// registry together. + #[test] + fn a_choice_naming_the_setup_runtime_finds_it_after_its_registry_entry_was_lost() -> Result<()> + { + type Choice = fn(&therock::InstalledRuntimeManifest, &mut RocmCliConfig) -> Option; + let choices: [(&str, Choice); 4] = [ + ("explicit key", |manifest, _| { + Some(manifest.runtime_key.clone()) + }), + ("explicit id", |manifest, _| { + Some(manifest.runtime_id.clone()) + }), + ("active runtime", |manifest, config| { + config.active_runtime_key = Some(manifest.runtime_key.clone()); + None + }), + ("configured default", |manifest, config| { + config.default_runtime_id = Some(manifest.runtime_id.clone()); + None + }), + ]; + for (label, choice) in choices { + for recovery in [SetupRecovery::PreviewOnly, SetupRecovery::Persist] { + let paths = test_paths(&format!( + "comfyui-choice-setup-{}-{recovery:?}", + label.replace(' ', "-") + )); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let mut config = RocmCliConfig::load(&paths)?; + let selector = choice(&manifest, &mut config); + + let selected = select_runtime(&paths, &config, selector.as_deref(), recovery) + .with_context(|| format!("{label}, {recovery:?}"))?; + + assert_eq!( + selected.manifest.runtime_key, manifest.runtime_key, + "{label}, {recovery:?}" + ); + let expected_entries = match recovery { + SetupRecovery::PreviewOnly => Vec::new(), + SetupRecovery::Persist => vec![format!("{}.json", manifest.runtime_key)], + }; + assert_eq!( + runtime_registry_entry_names(&paths), + expected_entries, + "{label}, {recovery:?}" + ); + } + } + Ok(()) + } + + /// What the recovery must not turn into: a choice that names some other + /// runtime is answered by the registry whether or not setup's runtime is + /// usable, a name that matches nothing is still refused (with the setup + /// runtime among those found when it is usable), and an unusable setup + /// runtime leaves the registry's own refusal standing instead of replacing + /// it with an error about a runtime the user did not ask for. + #[test] + fn recovering_the_setup_runtime_does_not_change_the_answer_for_other_choices() -> Result<()> { + let paths = test_paths("comfyui-choice-other-than-setup"); + let setup = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let other = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry = runtime_registry_entry_names(&paths); + assert_eq!(registry, [format!("{}.json", other.runtime_key)]); + + let picked = select_runtime( + &paths, + &config, + Some(&other.runtime_key), + SetupRecovery::PreviewOnly, + )?; + assert_eq!(picked.manifest.runtime_key, other.runtime_key); + + let miss = select_runtime( + &paths, + &config, + Some("no-such-runtime"), + SetupRecovery::PreviewOnly, + ) + .expect_err("nothing is called that") + .to_string(); + assert!( + miss.contains("ROCm runtime not found: `no-such-runtime`") + && miss.contains(&setup.runtime_key), + "{miss}" + ); + assert_eq!(runtime_registry_entry_names(&paths), registry); + + fs::remove_file( + setup + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + for recovery in [SetupRecovery::PreviewOnly, SetupRecovery::Persist] { + let picked = select_runtime(&paths, &config, Some(&other.runtime_key), recovery) + .with_context(|| format!("an unusable setup runtime, {recovery:?}"))?; + assert_eq!(picked.manifest.runtime_key, other.runtime_key); + let miss = select_runtime(&paths, &config, Some("no-such-runtime"), recovery) + .expect_err("nothing is called that") + .to_string(); + assert!( + miss.contains("ROCm runtime not found: `no-such-runtime`") + && !miss.contains("is not usable"), + "{recovery:?}: {miss}" + ); + assert_eq!( + runtime_registry_entry_names(&paths), + registry, + "{recovery:?}" + ); + } + Ok(()) + } + /// The two paths that pick "the one ready runtime" must not drift apart: /// for each registry state below, ComfyUI selects a runtime exactly when /// the engine paths do, and the same one. Both reach the rule through diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 7a1e63a27a..0a067bf6ee 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -501,12 +501,27 @@ pub(crate) fn select_single_ready_runtime( config: &RocmCliConfig, recovery: SetupRecovery, ) -> Result { + Ok(classify_single_ready_runtime( + registered_runtimes_with_setup(paths, config, recovery)?, + )) +} + +/// Every runtime the registry holds, with the one `rocm setup` created counted +/// even if its registry entry was lost. With [`SetupRecovery::Persist`] the +/// entry is written back first (`recover_setup_runtime_registration`); with +/// [`SetupRecovery::PreviewOnly`] nothing is written, and the setup runtime is +/// added from its own manifest as the registry would hold it once restored, so +/// both return the same list. An error when the setup runtime exists but +/// cannot be used. +pub(crate) fn registered_runtimes_with_setup( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, +) -> Result> { match recovery { SetupRecovery::Persist => { recover_setup_runtime_registration(paths, config)?; - Ok(classify_single_ready_runtime( - therock::load_runtime_manifests(paths)?, - )) + therock::load_runtime_manifests(paths) } SetupRecovery::PreviewOnly => { let setup = find_setup_runtime(paths, config)?; @@ -524,7 +539,7 @@ pub(crate) fn select_single_ready_runtime( ); sort_manifests_newest_install_first(&mut registered); } - Ok(classify_single_ready_runtime(registered)) + Ok(registered) } } } diff --git a/docs/architecture.md b/docs/architecture.md index 57460d7a8a..af97e39eaf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. - [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. - [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) — the argv helpers that `chat.rs`, `tool_policy.rs` and the freeform planner share, in a leaf module whose only `crate::` dependency is the leaf [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs): `chat_cli_arg_value`, `chat_cli_arg_value_checked`, `chat_cli_has_flag` and `ensure_flag` (from `chat.rs`), and `format_structured_tool_call` (which quotes each argument with `shell_quote`), `serve_args_request_cpu_device` and `device_policy_is_cpu_mode` (from `freeform_planner.rs`). They moved so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `ensure_flag` (read by `freeform_execution.rs`) and `freeform_planner.rs` re-exports `format_structured_tool_call`, so every other caller keeps its path. -- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Its `SetupRecovery` argument says whether recovering the setup runtime may write its lost registry entry back (`Persist`, what the engine paths and a real ComfyUI install use) or only count the runtime from its own manifest and write nothing (`PreviewOnly`, for a `--dry-run` preview, which must name the runtime the real run will use without changing a file); `SetupRecovery::for_dry_run` maps the flag. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. +- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Its `SetupRecovery` argument says whether recovering the setup runtime may write its lost registry entry back (`Persist`, what the engine paths and a real ComfyUI install use) or only count the runtime from its own manifest and write nothing (`PreviewOnly`, for a `--dry-run` preview, which must name the runtime the real run will use without changing a file); `SetupRecovery::for_dry_run` maps the flag. `registered_runtimes_with_setup` is the list that rule counts from (the registry plus the setup runtime, written back or not as `SetupRecovery` says), which `comfyui/runtime_env.rs` also searches when a choice that names a runtime is not answered by the registry alone. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing them (`write_runtime_manifest`, the one routine every way of registering a runtime goes through: an SDK install, `rocm runtimes import`, `rocm runtimes adopt` and the recovery of the runtime `rocm setup` created. It writes the registry entry, and for a runtime ROCm CLI owns (`InstalledRuntimeManifest::owns_install_root`, the same rule uninstall and prune use to decide whether they may delete the folder) the `.rocm-cli-runtime.json` in its install root; an imported or adopted runtime is someone else's folder, so it gets the registry entry only. Its one parameter is `ManifestOverwrite`: an install replaces its own entry, import and adopt refuse an existing one unless `--replace` is given, and recovery refuses so it only fills a missing entry. Neither file is written atomically) and loading them (`load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it; the tarball one asks `rocm_core::install_root_has_payload` whether the root holds a payload). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. @@ -115,7 +115,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: - [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). - [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. - [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. +- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. A choice that does name a runtime (`--runtime-id`, the active runtime, the configured default) is looked up in the registry first (`pick_from_registry`); only when the registry alone cannot answer is it looked up again among the registry plus the runtime `rocm setup` created, so the `--runtime-id` an install preview prints works from the state the preview left, and an unusable setup runtime leaves the registry's own refusal standing. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. - [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. - [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. diff --git a/docs/testing.md b/docs/testing.md index ec1e2c13f9..014f4904b9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -780,16 +780,20 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. -`comfyui-15` and `comfyui-16` cover which runtime `rocm comfyui install` uses +`comfyui-15` to `comfyui-17` cover which runtime `rocm comfyui install` uses when none is chosen: it recovers the runtime `rocm setup` created before counting the ready ones, as `rocm serve` does, so a runtime whose registry entry was lost is found. The real run (`comfyui-16`, against an existing checkout that needs no dependencies) writes the entry back; the `--dry-run` preview (`comfyui-15`) names the same runtime but writes nothing, because a preview -promises not to change files. Both run with no GPU: +promises not to change files. The preview prints the install command with that +runtime's `--runtime-id`, and `comfyui-17` runs that command as printed from the +state the preview left: an explicit `--runtime-id`, like an active or default +runtime, finds the runtime setup created when the registry alone cannot answer, +so the command works and writes the entry back. All three run with no GPU: ```bash -cargo xtask e2e -- -n "comfyui-1[56]" +cargo xtask e2e -- -n "comfyui-1[567]" ``` `comfyui-14` is different: it installs ComfyUI into a real, isolated managed diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index 9aee470019..c0a4517db3 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -339,7 +339,8 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv # names the same runtime but leaves the registry as it found it, because it # promises not to change files (this scenario). The preview prints the # `--runtime-id` of the command it would run, which is how it shows the - # choice. No GPU is needed: readiness is filesystem and manifest state. + # choice; comfyui-17 runs that command as printed. No GPU is needed: readiness + # is filesystem and manifest state. @id:comfyui-install-finds-the-runtime-setup-created Scenario: comfyui-15 - A ComfyUI install preview finds the runtime setup created without writing the registry Given a ready ROCm runtime that setup created and the registry has forgotten @@ -359,3 +360,19 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv When the user installs ComfyUI without choosing a runtime Then the install uses the runtime setup created And the runtime registry has an entry for it again + + # The preview's `install command:` line is how it shows the choice, so it has + # to be a command that works from the state the preview left, where the + # registry still has no entry for the runtime it named. Its `--runtime-id` is + # an explicit choice, which finds the runtime setup created the same way a + # missing choice does, instead of refusing with "Install ROCm first". Run as + # printed, the command installs into that runtime and writes the entry back. + @id:comfyui-install-preview-command-works-as-printed + Scenario: comfyui-17 - The install command a ComfyUI install preview prints works from the state the preview left + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + When the user previews a ComfyUI install without choosing a runtime + Then the runtime registry still has no entry for it + When the user runs the install command the preview printed + Then the install uses the runtime setup created + And the runtime registry has an entry for it again diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index 98f75637aa..54d564c851 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,14 +34,16 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! -//! `comfyui-15` and `comfyui-16` cover the other half of that selection: with +//! `comfyui-15` to `comfyui-17` cover the other half of that selection: with //! nothing chosen, the install recovers the runtime `rocm setup` created before //! it counts the ready ones, as `rocm serve` does, so a runtime the registry //! lost is found. They plant a ready runtime and the setup folder that names it, //! but no registry entry. `comfyui-15` previews the install with `--dry-run`, //! which prints the `--runtime-id` of the runtime it chose and writes nothing, //! so the registry still has no entry afterwards; `comfyui-16` runs the install -//! against an existing checkout and finds the entry written back. +//! against an existing checkout and finds the entry written back; `comfyui-17` +//! runs the command the preview printed, from the state the preview left, and +//! finds the same entry written back. //! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for @@ -498,6 +500,34 @@ async fn install_comfyui_without_runtime(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// Runs the `install command:` line of the preview in `world.cli_output`, as a +/// user would copy it: every word after `rocm` is an argument. The runtime key +/// planted for these scenarios needs no shell quoting, so splitting on +/// whitespace is the whole parse; the first word is checked so a line in +/// another shape fails here rather than running something else. +#[when("the user runs the install command the preview printed")] +async fn run_the_install_command_the_preview_printed(world: &mut E2eWorld) { + let stdout = world.cli_output.clone().unwrap_or_default(); + let line = stdout + .lines() + .find_map(|line| line.trim().strip_prefix("install command:")) + .unwrap_or_else(|| panic!("the preview printed no install command:\n{stdout}")); + let words: Vec<&str> = line.split_whitespace().collect(); + assert_eq!( + words.first().copied(), + Some("rocm"), + "the printed command should start with `rocm`: {line}" + ); + assert!( + words.contains(&"--runtime-id") && !words.contains(&"--dry-run"), + "the printed command should name its runtime and not preview: {line}" + ); + let (stdout, stderr, rc) = crate::run_rocm(world, &words[1..]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[then("ComfyUI install is refused as ambiguous")] async fn comfyui_install_refused(world: &mut E2eWorld) { let text = refusal_text(world); From f5d730b7b590e063a9ca042196020553fa7b53e2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 09:29:51 +0000 Subject: [PATCH 15/20] fix(comfyui): report a registry that cannot take the setup runtime back instead of refusing A choice that names a runtime (`--runtime-id`, the active runtime, the configured default) finds the runtime `rocm setup` created when its registry entry was lost. Looking it up discarded every error from that second look, the failure to write the entry back included. With the registry readable but not writable, `rocm comfyui install --dry-run` succeeded, and running the command it printed answered "Install ROCm first" (or "runtime not found") about a runtime that was there, instead of saying the entry could not be restored. The setup runtime is now found and picked without writing, whichever recovery was asked for, and a real run writes the entry back only for a pick it is going to use, propagating that error as it is. Not finding a usable setup runtime still leaves the registry's own refusal, so a choice naming something else and an unusable setup runtime answer as before. The engine paths already propagated the same failure (`rocm serve` and `rocm engines install` through `resolve_runtime_selector_to_exact_key` and the one-ready-runtime rule); nothing pinned it, so a unit test now covers the explicit and the default selection there. Covered by unit tests for each way of naming the runtime, with the preview succeeding and the real run reporting the error and leaving the registry as it was, and for a read-only registry directory holding other runtimes; and by scenario comfyui-18, which runs the printed command against a registry that cannot be written. The fixture is a plain file where the registry directory should be, because a read-only directory does not stop root. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 169 +++++++++++++++++- apps/rocm/src/engine_runtime_select.rs | 42 +++++ apps/rocm/src/test_support.rs | 15 ++ docs/architecture.md | 2 +- docs/testing.md | 13 +- tests/e2e-cucumber/features/comfyui.feature | 20 +++ tests/e2e-cucumber/tests/e2e/comfyui_steps.rs | 54 +++++- 7 files changed, 305 insertions(+), 10 deletions(-) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 24663f3975..91d8ff30e4 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -21,7 +21,8 @@ use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; use crate::runtime_adopt::{ - SetupRecovery, SingleReadyRuntime, registered_runtimes_with_setup, select_single_ready_runtime, + SetupRecovery, SingleReadyRuntime, recover_setup_runtime_registration, + registered_runtimes_with_setup, select_single_ready_runtime, }; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; @@ -131,6 +132,14 @@ fn format_runtime_statuses<'a>( /// runtime to the list without writing its entry, so the `--runtime-id` an /// install preview prints is a command that works from the state the preview /// left. If setup's runtime cannot be used, the registry's own answer stands. +/// +/// The two failures are not alike. Not finding a usable setup runtime leaves the +/// registry's refusal, because the user named something else or setup left +/// nothing. Failing to write the entry of a setup runtime that was found and +/// picked is not a refusal about a missing runtime, and swallowing it would +/// answer "Install ROCm first" about a runtime that is there, so that error is +/// returned as it is. The runtime is therefore found and picked without writing, +/// and written back only for a pick that is going to be used. fn pick_from_registry( paths: &AppPaths, config: &RocmCliConfig, @@ -148,10 +157,18 @@ fn pick_from_registry( Err(error) => error, } }; - match registered_runtimes_with_setup(paths, config, recovery) { - Ok(with_setup) if with_setup.len() > registered.len() => pick(&with_setup).cloned(), - _ => Err(refusal), + // Found without writing, whichever recovery was asked for: the write comes + // after the pick, so its failure is reported as what it is. + let with_setup = match registered_runtimes_with_setup(paths, config, SetupRecovery::PreviewOnly) + { + Ok(with_setup) if with_setup.len() > registered.len() => with_setup, + _ => return Err(refusal), + }; + let picked = pick(&with_setup)?.clone(); + if recovery == SetupRecovery::Persist { + recover_setup_runtime_registration(paths, config)?; } + Ok(picked) } fn install_rocm_first() -> anyhow::Error { @@ -1258,6 +1275,150 @@ mod tests { Ok(()) } + /// Finding the setup runtime for a named choice must not hide that its + /// registry entry cannot be written back. The preview of such an install + /// succeeds, because it writes nothing; the printed command is the real run, + /// and it has to say why it cannot restore the entry rather than answer as if + /// there were no runtime ("Install ROCm first"). Each row asserts the error + /// and the registry together: the failing run leaves the registry as it + /// found it, and the runtime was still found (the preview, and the + /// unwritable registry's own refusal for a name that is not setup's). + #[test] + fn a_choice_naming_the_setup_runtime_reports_a_registry_that_cannot_take_it_back() -> Result<()> + { + type Choice = fn(&therock::InstalledRuntimeManifest, &mut RocmCliConfig) -> Option; + let choices: [(&str, Choice); 5] = [ + ("explicit key", |manifest, _| { + Some(manifest.runtime_key.clone()) + }), + ("explicit id", |manifest, _| { + Some(manifest.runtime_id.clone()) + }), + ("active runtime", |manifest, config| { + config.active_runtime_key = Some(manifest.runtime_key.clone()); + None + }), + ("configured default", |manifest, config| { + config.default_runtime_id = Some(manifest.runtime_id.clone()); + None + }), + // Not a named choice, but the same persistence, reported the same way. + ("no choice", |_, _| None), + ]; + for (label, choice) in choices { + let paths = test_paths(&format!( + "comfyui-choice-unwritable-{}", + label.replace(' ', "-") + )); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let mut config = RocmCliConfig::load(&paths)?; + let selector = choice(&manifest, &mut config); + let registry = crate::test_support::make_runtime_registry_unwritable(&paths)?; + + let previewed = select_runtime( + &paths, + &config, + selector.as_deref(), + SetupRecovery::PreviewOnly, + ) + .with_context(|| format!("{label}: the preview writes nothing"))?; + assert_eq!( + previewed.manifest.runtime_key, manifest.runtime_key, + "{label}" + ); + + let error = + select_runtime(&paths, &config, selector.as_deref(), SetupRecovery::Persist) + .expect_err("a registry that cannot be written cannot take the entry back"); + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + manifest.runtime_key + )), + "{label}: got: {message}" + ); + assert!( + !message.contains("Install ROCm first") && !message.contains("not found"), + "{label}: the persistence error must not be replaced by a refusal about a missing runtime: {message}" + ); + assert_eq!( + fs::read_to_string(®istry)?, + "not a directory", + "{label}: a failed restore leaves the registry as it was" + ); + } + Ok(()) + } + + /// The same, with the registry readable and holding another runtime, which is + /// the state a read-only registry directory leaves. The refusal that stands + /// without the fix is then the registry's "not found", so the error is + /// compared with that. Where the directory stays writable regardless (an + /// administrator), the write succeeds and the test checks the restore + /// instead, so it cannot pass by writing nothing. + #[cfg(unix)] + #[test] + fn a_read_only_registry_directory_is_reported_not_hidden_behind_a_not_found() -> Result<()> { + use crate::runtimes_cmd::runtime_registry_dir; + use std::os::unix::fs::PermissionsExt; + + let paths = test_paths("comfyui-choice-read-only-registry"); + let setup = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry = runtime_registry_dir(&paths); + let original = fs::metadata(®istry)?.permissions(); + fs::set_permissions(®istry, fs::Permissions::from_mode(0o555))?; + + let selected = select_runtime( + &paths, + &config, + Some(&setup.runtime_key), + SetupRecovery::Persist, + ); + + fs::set_permissions(®istry, original)?; + match selected { + Err(error) => { + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + setup.runtime_key + )) && !message.contains("not found"), + "got: {message}" + ); + assert!( + !runtime_registry_entry_names(&paths) + .contains(&format!("{}.json", setup.runtime_key)), + "a failed restore writes no entry" + ); + } + Ok(selected) => { + assert_eq!(selected.manifest.runtime_key, setup.runtime_key); + assert!( + runtime_registry_entry_names(&paths) + .contains(&format!("{}.json", setup.runtime_key)), + "the directory stayed writable, so the entry is restored" + ); + } + } + Ok(()) + } + /// What the recovery must not turn into: a choice that names some other /// runtime is answered by the registry whether or not setup's runtime is /// usable, a name that matches nothing is still refused (with the setup diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 061f968b49..73589e7967 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -1139,4 +1139,46 @@ mod tests { let _ = std::fs::remove_dir_all(root); Ok(()) } + + /// Recovering the setup runtime writes its registry entry, and a registry + /// that cannot take it is reported as that on every path that recovers: + /// an explicit `--runtime-id` and the one-ready-runtime default (which + /// `serve` and `engines install` both reach). Neither may answer as if there + /// were no runtime, or carry on without one: the runtime is there, and the + /// failure is what stops the command. The error and the registry are + /// asserted together. + #[test] + fn engine_selection_reports_a_registry_that_cannot_take_the_setup_runtime_back() -> Result<()> { + for explicit in [true, false] { + let (root, paths) = test_paths("setup-recovery-unwritable-registry"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let registry = crate::test_support::make_runtime_registry_unwritable(&paths)?; + let runtime_id = explicit.then_some(manifest.runtime_key.as_str()); + + let error = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::load(&paths)?, "vllm", runtime_id, None), + ) + .expect_err("a registry that cannot be written cannot take the entry back"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "failed to restore setup runtime `{}`", + manifest.runtime_key + )), + "explicit: {explicit}, got: {message}" + ); + assert_eq!( + std::fs::read_to_string(®istry)?, + "not a directory", + "explicit: {explicit}: a failed restore leaves the registry as it was" + ); + let _ = std::fs::remove_dir_all(root); + } + Ok(()) + } } diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 9cbb22aa61..e4f0d406cb 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -252,6 +252,21 @@ pub(crate) fn write_test_setup_runtime_missing_from_registry( Ok(manifest) } +/// Make the runtime registry impossible to write for any user, an administrator +/// included: a plain file stands where its directory should be, so listing the +/// registry finds nothing and creating it fails. A read-only directory would +/// not do, because it does not stop `root`, which is who the e2e suite can run as. +/// Returns the path of that file, so a test can show it was left as it was. +pub(crate) fn make_runtime_registry_unwritable(paths: &AppPaths) -> Result { + let registry = runtime_registry_dir(paths); + let _ = fs::remove_dir_all(®istry); + if let Some(parent) = registry.parent() { + fs::create_dir_all(parent)?; + } + fs::write(®istry, "not a directory")?; + Ok(registry) +} + /// File names in the runtime registry directory, sorted; empty when the /// directory does not exist. Lets a test assert that a selection did, or did /// not, write a registry entry. diff --git a/docs/architecture.md b/docs/architecture.md index af97e39eaf..4598feea04 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -115,7 +115,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: - [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). - [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. - [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. A choice that does name a runtime (`--runtime-id`, the active runtime, the configured default) is looked up in the registry first (`pick_from_registry`); only when the registry alone cannot answer is it looked up again among the registry plus the runtime `rocm setup` created, so the `--runtime-id` an install preview prints works from the state the preview left, and an unusable setup runtime leaves the registry's own refusal standing. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. +- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. A choice that does name a runtime (`--runtime-id`, the active runtime, the configured default) is looked up in the registry first (`pick_from_registry`); only when the registry alone cannot answer is it looked up again among the registry plus the runtime `rocm setup` created, so the `--runtime-id` an install preview prints works from the state the preview left, and an unusable setup runtime leaves the registry's own refusal standing. The setup runtime is found and picked without writing, and a real run writes its entry back only for a pick it is going to use, so a registry that cannot take the entry is reported as that failure rather than as a missing runtime. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. - [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. - [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. diff --git a/docs/testing.md b/docs/testing.md index 014f4904b9..3045737ff8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -780,7 +780,7 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. -`comfyui-15` to `comfyui-17` cover which runtime `rocm comfyui install` uses +`comfyui-15` to `comfyui-18` cover which runtime `rocm comfyui install` uses when none is chosen: it recovers the runtime `rocm setup` created before counting the ready ones, as `rocm serve` does, so a runtime whose registry entry was lost is found. The real run (`comfyui-16`, against an existing checkout that @@ -790,10 +790,17 @@ promises not to change files. The preview prints the install command with that runtime's `--runtime-id`, and `comfyui-17` runs that command as printed from the state the preview left: an explicit `--runtime-id`, like an active or default runtime, finds the runtime setup created when the registry alone cannot answer, -so the command works and writes the entry back. All three run with no GPU: +so the command works and writes the entry back. `comfyui-18` runs the same +printed command when the registry cannot take that entry back (a plain file +stands where its directory should be, which also stops `root`; a read-only +directory would not): the preview still succeeds, and the real run fails naming +the runtime whose entry it could not restore, rather than saying "Install ROCm +first", and leaves the registry as it was. The variant with a read-only registry +directory that holds other runtimes is a unit test, because a read-only +directory is writable to `root`. All four run with no GPU: ```bash -cargo xtask e2e -- -n "comfyui-1[567]" +cargo xtask e2e -- -n "comfyui-1[5678]" ``` `comfyui-14` is different: it installs ComfyUI into a real, isolated managed diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index c0a4517db3..481fe22329 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -376,3 +376,23 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv When the user runs the install command the preview printed Then the install uses the runtime setup created And the runtime registry has an entry for it again + + # The same printed command, when the registry cannot take the entry back (a + # file stands where its directory should be, which stops an administrator as + # well; a read-only directory would not). The preview writes nothing, so it + # still succeeds and names the runtime. The real run finds that runtime the + # same way, fails to restore its entry, and says so, rather than answering + # "Install ROCm first" as if setup had left no runtime. The registry is left as + # it was. A read-only registry directory with other runtimes in it, the other + # way to reach this, is covered by a unit test: the suite can run as root, for + # whom a read-only directory is writable. + @id:comfyui-install-reports-a-registry-it-cannot-write + Scenario: comfyui-18 - The install command a ComfyUI install preview prints reports a registry that cannot take the runtime back + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + And the runtime registry cannot be written + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects that runtime + When the user runs the install command the preview printed + Then the install is refused because the runtime's registry entry could not be restored + And the runtime registry is as it was diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index 54d564c851..9fbeeffc04 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,7 +34,7 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! -//! `comfyui-15` to `comfyui-17` cover the other half of that selection: with +//! `comfyui-15` to `comfyui-18` cover the other half of that selection: with //! nothing chosen, the install recovers the runtime `rocm setup` created before //! it counts the ready ones, as `rocm serve` does, so a runtime the registry //! lost is found. They plant a ready runtime and the setup folder that names it, @@ -43,7 +43,9 @@ //! so the registry still has no entry afterwards; `comfyui-16` runs the install //! against an existing checkout and finds the entry written back; `comfyui-17` //! runs the command the preview printed, from the state the preview left, and -//! finds the same entry written back. +//! finds the same entry written back. `comfyui-18` runs that command with a +//! registry that cannot be written and finds the failure to restore the entry +//! reported. //! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for @@ -686,6 +688,54 @@ async fn comfyui_checkout_needing_no_dependencies(world: &mut E2eWorld) { ); } +/// What stands where the registry directory should be for `comfyui-18`. A plain +/// file stops every user, an administrator included, from creating the registry, +/// where a read-only directory would not stop `root`. +const UNWRITABLE_REGISTRY_CONTENTS: &str = "not a directory"; + +#[given("the runtime registry cannot be written")] +async fn runtime_registry_cannot_be_written(world: &mut E2eWorld) { + let registry = registry_dir(world); + assert!( + !registry.exists(), + "the registry must not exist yet for this step to mean anything" + ); + write_fixture(®istry, UNWRITABLE_REGISTRY_CONTENTS); +} + +/// The failing install names the entry it could not restore, so the user learns +/// why, and is not told that setup left no runtime: the runtime exists, and the +/// preview just named it. +#[then("the install is refused because the runtime's registry entry could not be restored")] +async fn install_refused_for_the_registry_entry(world: &mut E2eWorld) { + let text = refusal_text(world); + assert!( + world.cli_rc.is_some_and(|rc| rc != 0), + "the install should fail when the registry cannot take the runtime back, got:\n{text}" + ); + assert!( + text.contains(&format!( + "failed to restore setup runtime `{SETUP_RUNTIME_KEY}`" + )), + "the failure should name the runtime whose entry could not be restored, got:\n{text}" + ); + assert!( + !text.contains("Install ROCm first"), + "the failure must not tell the user to install ROCm when setup left a runtime, got:\n{text}" + ); +} + +#[then("the runtime registry is as it was")] +async fn runtime_registry_is_as_it_was(world: &mut E2eWorld) { + let registry = registry_dir(world); + assert_eq!( + std::fs::read_to_string(®istry).ok().as_deref(), + Some(UNWRITABLE_REGISTRY_CONTENTS), + "a failed restore must leave {} as it found it", + registry.display() + ); +} + #[then("the install uses the runtime setup created")] async fn install_uses_the_setup_runtime(world: &mut E2eWorld) { let text = refusal_text(world); From fa5b41e7ce70f75a9b66889deec795d44abe27cc Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 11:49:28 +0000 Subject: [PATCH 16/20] fix(comfyui): set aside an unusable setup runtime while a registered runtime is ready With no runtime chosen, `rocm comfyui install` failed with the setup recovery error ("setup runtime ... is not usable") whenever the folder `rocm setup` named held a runtime that no longer passed the readiness check, even when exactly one other runtime was registered and ready. A choice that names a runtime already keeps the registry's answer in that case, so the two ways of asking disagreed, and the engine commands reached the same rule through the same function. A setup runtime that cannot be used is not a ready runtime and cannot change which runtimes are ready, so `select_single_ready_runtime` now counts the registry alone when the setup runtime is unusable: one ready runtime is picked, several are refused as ambiguous, and nothing is written. When the registry holds no ready runtime the setup runtime was the only candidate and its error is still the answer. A usable setup runtime whose registry entry cannot be written back is unchanged and still reported as that failure, because it is one of the runtimes being counted. The rule is shared, so `rocm serve` and `rocm engines install` pick the registered runtime in this corner too; their other recovery call sites are untouched. The state matrix is a test of its own: the choice (none, a registered runtime, the setup runtime) against 0, 1 or 2 ready registered runtimes, a setup folder that is absent, usable or unusable, a registry that takes writes or not, and preview or real run, each row asserting the message and the registry entries together. Scenario comfyui-19 covers the install preview with an unusable setup runtime beside one ready registered runtime. Signed-off-by: Jussi Elo --- apps/rocm/src/comfyui/runtime_env.rs | 315 ++++++++++++++++++ apps/rocm/src/runtime_adopt.rs | 125 ++++++- docs/architecture.md | 4 +- docs/testing.md | 11 +- tests/e2e-cucumber/features/comfyui.feature | 15 + tests/e2e-cucumber/tests/e2e/comfyui_steps.rs | 83 ++++- 6 files changed, 534 insertions(+), 19 deletions(-) diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 91d8ff30e4..5f726f0d28 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -1493,6 +1493,306 @@ mod tests { Ok(()) } + /// A registry directory that stops writing, as far as this process can be + /// stopped: a read-only directory, which does not stop `root`. `None` where it + /// would not bite (the process can write anyway, or the host has no such + /// modes), so a caller skips the row instead of passing for the wrong reason. + /// Restores the mode on drop. + #[cfg(unix)] + struct ReadOnlyRegistry { + path: PathBuf, + original: fs::Permissions, + } + + #[cfg(unix)] + impl Drop for ReadOnlyRegistry { + fn drop(&mut self) { + let _ = fs::set_permissions(&self.path, self.original.clone()); + } + } + + #[cfg(unix)] + fn make_registry_read_only(paths: &AppPaths) -> Option { + use std::os::unix::fs::PermissionsExt; + + let path = crate::runtimes_cmd::runtime_registry_dir(paths); + let original = fs::metadata(&path).ok()?.permissions(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o555)).ok()?; + let guard = ReadOnlyRegistry { path, original }; + let probe = guard.path.join(".write-probe"); + if fs::write(&probe, "").is_ok() { + let _ = fs::remove_file(probe); + return None; + } + Some(guard) + } + + #[cfg(not(unix))] + struct ReadOnlyRegistry; + + #[cfg(not(unix))] + fn make_registry_read_only(_paths: &AppPaths) -> Option { + None + } + + /// Every state `rocm comfyui install` can pick a runtime from, one row each: + /// what the user chose (nothing, a registered runtime by name, the setup + /// runtime by name) against how many ready runtimes the registry holds (0, 1, + /// 2), what the setup folder holds (nothing, a usable runtime whose registry + /// entry was lost, a runtime that is no longer usable), whether the registry + /// takes a new entry, and whether the install is a preview or a real run. + /// + /// The rules the rows pin: a choice that names a registered runtime is + /// answered by the registry whatever setup holds. With nothing chosen the + /// ready runtimes are counted, and a setup runtime that cannot be used is not + /// one of them: it does not stop a registered ready runtime from being picked + /// (one) or refused as ambiguous (two), and is reported only when it was the + /// last hope (none registered). A setup runtime that can be used is counted, + /// and a failure to write its entry back is reported as that, whatever else + /// the count would have said. Each row asserts the message and the registry + /// together. + #[test] + fn comfyui_runtime_selection_state_matrix() -> Result<()> { + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Chosen { + Nothing, + Registered, + SetupRuntime, + } + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum SetupFolder { + Absent, + Usable, + Unusable, + } + struct Expected { + selected: Option<&'static str>, + error_has: Vec, + error_lacks: &'static [&'static str], + setup_entry_written: bool, + } + + let registered_keys = ["runtime-a", "runtime-b"]; + const SETUP_KEY: &str = "runtime-s"; + const MULTIPLE: &str = "Multiple ROCm runtimes are ready"; + let restore_failed = format!("failed to restore setup runtime `{SETUP_KEY}`"); + let mut rows_run = 0; + let mut rows_skipped = 0; + + for chosen in [Chosen::Nothing, Chosen::Registered, Chosen::SetupRuntime] { + for ready in 0..=2usize { + for setup in [ + SetupFolder::Absent, + SetupFolder::Usable, + SetupFolder::Unusable, + ] { + for registry_writable in [true, false] { + for recovery in [SetupRecovery::Persist, SetupRecovery::PreviewOnly] { + if chosen == Chosen::Registered && ready == 0 { + continue; // nothing registered to name + } + let row = format!( + "{chosen:?}, {ready} ready, setup {setup:?}, writable {registry_writable}, {recovery:?}" + ); + let paths = test_paths(&format!( + "comfyui-selection-matrix-{}", + rows_run + rows_skipped + )); + for (index, key) in registered_keys.iter().take(ready).enumerate() { + write_test_pip_runtime( + &paths, + key, + "therock-release:gfx120X-all", + "7.14.0", + 10 + index as u128, + )?; + } + if setup != SetupFolder::Absent { + let manifest = + crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, SETUP_KEY, + )?; + if setup == SetupFolder::Unusable { + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + } + } + let config = RocmCliConfig::load(&paths)?; + // A registry that cannot take an entry: a plain file for + // an empty one (stops `root` too), a read-only directory + // where it holds runtimes. + let registry_path = crate::runtimes_cmd::runtime_registry_dir(&paths); + let _read_only; + if !registry_writable { + if ready == 0 { + crate::test_support::make_runtime_registry_unwritable(&paths)?; + } else { + match make_registry_read_only(&paths) { + Some(guard) => _read_only = guard, + None => { + rows_skipped += 1; + continue; + } + } + } + } + rows_run += 1; + + let persists = recovery == SetupRecovery::Persist; + let setup_usable = setup == SetupFolder::Usable; + let write_fails = persists && !registry_writable && setup_usable; + let writes = persists && registry_writable && setup_usable; + let ready_keys = registered_keys[..ready].to_vec(); + let expected = match chosen { + Chosen::Nothing => match (setup, ready) { + (SetupFolder::Usable, _) if write_fails => Expected { + selected: None, + error_has: vec![restore_failed.clone()], + error_lacks: &["Install ROCm first", MULTIPLE], + setup_entry_written: false, + }, + (SetupFolder::Usable, 0) => Expected { + selected: Some(SETUP_KEY), + error_has: vec![], + error_lacks: &[], + setup_entry_written: writes, + }, + (SetupFolder::Usable, _) => Expected { + selected: None, + error_has: std::iter::once(MULTIPLE.to_owned()) + .chain(ready_keys.iter().map(|key| (*key).to_owned())) + .chain([SETUP_KEY.to_owned()]) + .collect(), + error_lacks: &["Install ROCm first"], + setup_entry_written: writes, + }, + (SetupFolder::Unusable, 0) => Expected { + selected: None, + error_has: vec![ + format!("setup runtime `{SETUP_KEY}`"), + "is not usable".to_owned(), + ], + error_lacks: &["Install ROCm first"], + setup_entry_written: false, + }, + (SetupFolder::Absent, 0) => Expected { + selected: None, + error_has: vec!["Install ROCm first".to_owned()], + error_lacks: &["is not usable"], + setup_entry_written: false, + }, + (_, 1) => Expected { + selected: Some("runtime-a"), + error_has: vec![], + error_lacks: &[], + setup_entry_written: false, + }, + (_, _) => Expected { + selected: None, + error_has: std::iter::once(MULTIPLE.to_owned()) + .chain(ready_keys.iter().map(|key| (*key).to_owned())) + .collect(), + error_lacks: &["is not usable", SETUP_KEY], + setup_entry_written: false, + }, + }, + Chosen::Registered => Expected { + selected: Some("runtime-a"), + error_has: vec![], + error_lacks: &[], + setup_entry_written: false, + }, + Chosen::SetupRuntime => match setup { + SetupFolder::Usable if write_fails => Expected { + selected: None, + error_has: vec![restore_failed.clone()], + error_lacks: &["Install ROCm first", "not found"], + setup_entry_written: false, + }, + SetupFolder::Usable => Expected { + selected: Some(SETUP_KEY), + error_has: vec![], + error_lacks: &[], + setup_entry_written: writes, + }, + SetupFolder::Absent | SetupFolder::Unusable => Expected { + selected: None, + error_has: vec![if ready == 0 { + "Install ROCm first".to_owned() + } else { + format!("ROCm runtime not found: `{SETUP_KEY}`") + }], + error_lacks: &["is not usable"], + setup_entry_written: false, + }, + }, + }; + + let selector = match chosen { + Chosen::Nothing => None, + Chosen::Registered => Some("runtime-a"), + Chosen::SetupRuntime => Some(SETUP_KEY), + }; + let outcome = select_runtime(&paths, &config, selector, recovery); + + match (outcome, expected.selected) { + (Ok(selected), Some(key)) => { + assert_eq!(selected.manifest.runtime_key, key, "{row}") + } + (Ok(selected), None) => panic!( + "{row}: expected a refusal, picked {}", + selected.manifest.runtime_key + ), + (Err(error), Some(key)) => { + panic!("{row}: expected {key}, got: {error:#}") + } + (Err(error), None) => { + let message = format!("{error:#}"); + for needle in &expected.error_has { + assert!( + message.contains(needle.as_str()), + "{row}: `{needle}` missing from: {message}" + ); + } + for needle in expected.error_lacks { + assert!( + !message.contains(needle), + "{row}: `{needle}` must not appear in: {message}" + ); + } + } + } + + let mut entries: Vec = + ready_keys.iter().map(|key| format!("{key}.json")).collect(); + if expected.setup_entry_written { + entries.push(format!("{SETUP_KEY}.json")); + } + entries.sort(); + assert_eq!(runtime_registry_entry_names(&paths), entries, "{row}"); + if !registry_writable && ready == 0 { + assert_eq!( + fs::read_to_string(®istry_path)?, + "not a directory", + "{row}: a failed restore leaves the registry as it was" + ); + } + } + } + } + } + } + // The rows that need a registry holding runtimes that refuses a write skip + // themselves where the process can write anyway; the rest must have run. + assert_eq!(rows_run + rows_skipped, 96, "the matrix has 96 rows"); + assert!(rows_run >= 60, "ran {rows_run}, skipped {rows_skipped}"); + Ok(()) + } + /// The two paths that pick "the one ready runtime" must not drift apart: /// for each registry state below, ComfyUI selects a runtime exactly when /// the engine paths do, and the same one. Both reach the rule through @@ -1514,6 +1814,7 @@ mod tests { ReadyAndUnusable, SetupRuntimeLost, SetupRuntimeLostBesideAReadyOne, + UnusableSetupRuntimeBesideAReadyOne, } for (label, state, expected) in [ ("empty", State::Empty, None), @@ -1535,6 +1836,11 @@ mod tests { State::SetupRuntimeLostBesideAReadyOne, None, ), + ( + "unusable-setup-runtime-beside-a-ready-one", + State::UnusableSetupRuntimeBesideAReadyOne, + Some("runtime-a"), + ), ] { // Each path gets its own copy of the state: a recovery run by one // writes the registry, which would otherwise hand the other path a @@ -1579,6 +1885,15 @@ mod tests { "runtime-s", )?; } + State::UnusableSetupRuntimeBesideAReadyOne => { + plant("runtime-a")?; + break_runtime( + &crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?, + )?; + } } Ok(()) }; diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 0a067bf6ee..4df8aa13ad 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -420,13 +420,9 @@ fn find_setup_runtime(paths: &AppPaths, config: &RocmCliConfig) -> Result