diff --git a/apps/rocm/src/comfyui/install.rs b/apps/rocm/src/comfyui/install.rs index 0ac2a164de..6e6b598bbe 100644 --- a/apps/rocm/src/comfyui/install.rs +++ b/apps/rocm/src/comfyui/install.rs @@ -27,6 +27,7 @@ use super::{ }; use crate::cli_progress::AnimatedSpinner; use crate::freeform_planner::format_structured_tool_call; +use crate::runtime_adopt::SetupRecovery; use crate::therock; use anyhow::{Context, Result, bail}; use flate2::read::GzDecoder; @@ -48,7 +49,14 @@ pub(crate) fn install( options: ComfyUiInstallOptions, ) -> Result { paths.ensure()?; - let runtime = select_runtime(paths, config, options.runtime_id.as_deref())?; + // A preview names the runtime the real run will use but does not write the + // registry entry that run would restore. + let runtime = select_runtime( + paths, + config, + options.runtime_id.as_deref(), + SetupRecovery::for_dry_run(options.dry_run), + )?; let app_root = runtime_app_root(&runtime.manifest); let source_path = source_path_from_app_root(&app_root); let log_path = install_log_path_from_app_root(&app_root); @@ -1908,6 +1916,102 @@ mod tests { remove_test_root(&paths); Ok(()) } + + /// `--dry-run` promises not to change files. With nothing chosen, the install + /// finds the runtime `rocm setup` created even though its registry entry was + /// lost, but a preview must not write that entry back: it names the same + /// runtime the real run will use and leaves the registry as it found it. + #[test] + fn dry_run_previews_the_setup_runtime_without_restoring_its_registry_entry() -> Result<()> { + let paths = test_paths("comfyui-dry-run-setup-runtime"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + + let rendered = install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: true, + }, + )?; + + assert!(rendered.contains(" mode: dry-run\n"), "{rendered}"); + assert!( + rendered.contains(&format!( + " ROCm runtime: {}\n", + therock::runtime_version_display(&manifest.version) + )), + "{rendered}" + ); + assert!( + rendered + .lines() + .any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {}", manifest.runtime_key))), + "{rendered}" + ); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "a preview must not restore the registry entry" + ); + remove_test_root(&paths); + Ok(()) + } + + /// The other half: the real run does restore the entry, so the runtime the + /// preview named is one `rocm runtimes list` shows once the install has run. + #[cfg(unix)] + #[test] + fn install_restores_the_setup_runtime_registry_entry_the_preview_left_alone() -> Result<()> { + let paths = test_paths("comfyui-install-setup-runtime"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + let app_root = runtime_app_root(&manifest); + plant_cached_source_archive( + &app_root, + &[("main.py", "new main"), ("requirements.txt", "torch\n")], + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("runtime has no Python")?, + ); + rocm_core::fake_executable::write_fake_executable( + &python, + "#!/bin/sh\nprintf '%s' '{\"torch_version\": \"2.4.0\", \"torch_cuda_available\": true, \"device_count\": 1, \"devices\": [\"Fake GPU\"]}' > \"$2\"\n", + )?; + + install( + &paths, + &config, + ComfyUiInstallOptions { + runtime_id: None, + reinstall: false, + dry_run: false, + }, + )?; + + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + remove_test_root(&paths); + Ok(()) + } } /// Property tests for the ComfyUI source unpack, judged on the filesystem by diff --git a/apps/rocm/src/comfyui/runtime_env.rs b/apps/rocm/src/comfyui/runtime_env.rs index 3e496900be..ee95deafde 100644 --- a/apps/rocm/src/comfyui/runtime_env.rs +++ b/apps/rocm/src/comfyui/runtime_env.rs @@ -17,6 +17,7 @@ //! ComfyUI types. use super::{ComfyUiManifest, ComfyUiRuntimeEnvironment, SelectedRuntime}; +use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; use crate::runtimes_cmd::runtime_usability_status; use crate::therock; use anyhow::{Context, Result, bail}; @@ -32,16 +33,13 @@ pub(super) fn select_runtime( paths: &AppPaths, config: &RocmCliConfig, selector: Option<&str>, + recovery: SetupRecovery, ) -> Result { - let manifests = therock::load_runtime_manifests(paths)?; - if manifests.is_empty() { - bail!( - "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." - ); - } let manifest = match selector.map(str::trim).filter(|value| !value.is_empty()) { - Some(selector) => select_runtime_by_selector(&manifests, selector)?.clone(), - None => select_default_runtime(config, &manifests)?.clone(), + Some(selector) => { + select_runtime_by_selector(®istered_runtimes(paths)?, selector)?.clone() + } + None => select_default_runtime(paths, config, recovery)?, }; if runtime_usability_status(&manifest) != "ready" { bail!( @@ -119,16 +117,44 @@ fn format_runtime_statuses<'a>( .join(", ") } -fn select_default_runtime<'a>( +/// Every registered runtime, or the refusal to install ROCm when there is none. +fn registered_runtimes(paths: &AppPaths) -> Result> { + let manifests = therock::load_runtime_manifests(paths)?; + if manifests.is_empty() { + return Err(install_rocm_first()); + } + Ok(manifests) +} + +fn install_rocm_first() -> anyhow::Error { + anyhow::anyhow!( + "Install ROCm first: run `rocm` and pick \"Set up this system\", then install ComfyUI." + ) +} + +fn select_default_runtime( + paths: &AppPaths, config: &RocmCliConfig, - manifests: &'a [therock::InstalledRuntimeManifest], -) -> Result<&'a therock::InstalledRuntimeManifest> { + recovery: SetupRecovery, +) -> Result { if let Some(active_key) = config.active_runtime_key.as_deref() { - return select_runtime_by_selector(manifests, active_key); + return select_runtime_by_selector(®istered_runtimes(paths)?, active_key).cloned(); } - let Some(default_runtime_id) = config.default_runtime_id.as_deref() else { - return select_single_ready_runtime(manifests); - }; + match config.default_runtime_id.as_deref() { + Some(default_runtime_id) => { + select_runtime_by_default_id(®istered_runtimes(paths)?, default_runtime_id).cloned() + } + // Nothing says which runtime to use: the rule the engine paths share. + None => { + single_ready_runtime_or_refusal(select_single_ready_runtime(paths, config, recovery)?) + } + } +} + +fn select_runtime_by_default_id<'a>( + manifests: &'a [therock::InstalledRuntimeManifest], + default_runtime_id: &str, +) -> Result<&'a therock::InstalledRuntimeManifest> { let matches = manifests .iter() .filter(|manifest| manifest.runtime_id.eq_ignore_ascii_case(default_runtime_id)) @@ -156,23 +182,24 @@ fn select_default_runtime<'a>( } } -fn select_single_ready_runtime( - manifests: &[therock::InstalledRuntimeManifest], -) -> Result<&therock::InstalledRuntimeManifest> { - let ready = manifests - .iter() - .filter(|manifest| runtime_usability_status(manifest) == "ready") - .collect::>(); - match ready.as_slice() { - [manifest] => Ok(*manifest), - [] => { - let statuses = format_runtime_statuses(manifests); +/// ComfyUI's words for what [`select_single_ready_runtime`] found. The rule +/// itself, including which runtimes count as ready, is not decided here. +fn single_ready_runtime_or_refusal( + found: SingleReadyRuntime, +) -> Result { + match found { + SingleReadyRuntime::One(manifest) => Ok(*manifest), + SingleReadyRuntime::Zero { registered } if registered.is_empty() => { + Err(install_rocm_first()) + } + SingleReadyRuntime::Zero { registered } => { + let statuses = format_runtime_statuses(®istered); bail!( "No ROCm runtime is ready. Fix one or install another: run `rocm` and pick \"Set up this system\" (or run `rocm install sdk`), then install ComfyUI. Runtimes found: {statuses}. See `rocm runtimes list`." ) } - _ => { - let available = format_available_runtime_keys(ready.iter().copied()); + SingleReadyRuntime::Many { ready } => { + let available = format_available_runtime_keys(&ready); bail!( "Multiple ROCm runtimes are ready. Pick one in `/runtimes`, set a default with `rocm runtimes activate `, or pass `--runtime-id `. Available: {available}. See `rocm runtimes list`." ) @@ -397,9 +424,22 @@ mod tests { use super::*; use crate::comfyui::render_status; use crate::comfyui::tests::{ready_runtime_manifest, test_paths, write_runtime_manifest}; + use crate::runtime_adopt::classify_single_ready_runtime; + use crate::test_support::{runtime_registry_entry_names, write_test_pip_runtime}; use rocm_core::{runtime_python_executable_in_env, runtime_rocm_library_filename}; use std::fs; + /// `select_runtime` as a real install runs it: a runtime that only setup + /// recovery can make visible has its registry entry written back. The + /// preview mode has its own tests. + fn persisting_select_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + selector: Option<&str>, + ) -> Result { + select_runtime(paths, config, selector, SetupRecovery::Persist) + } + #[test] fn default_runtime_selection_uses_single_ready_runtime() -> Result<()> { let paths = test_paths("comfyui-single-ready-runtime"); @@ -461,7 +501,8 @@ mod tests { }; let manifests = [manifest]; - let selected = select_default_runtime(&RocmCliConfig::default(), &manifests)?; + let selected = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec()))?; assert_eq!( selected.runtime_key, "release-pip-gfx120x-all-7-13-0a20260511" @@ -510,8 +551,9 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("two ready runtimes should be ambiguous"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("two ready runtimes should be ambiguous"); let message = error.to_string(); assert_actionable(&message); @@ -545,8 +587,9 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; let manifests = [unusable]; - let error = select_default_runtime(&RocmCliConfig::default(), &manifests) - .expect_err("an unusable runtime is not ready"); + let error = + single_ready_runtime_or_refusal(classify_single_ready_runtime(manifests.to_vec())) + .expect_err("an unusable runtime is not ready"); let message = error.to_string(); assert!( @@ -628,7 +671,7 @@ mod tests { ); // `select_runtime`'s bail when nothing is registered at all. - let error = select_runtime(&paths, &RocmCliConfig::default(), None) + let error = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) .expect_err("no registered runtime means no selection"); let message = error.to_string(); assert!( @@ -653,11 +696,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("no-such-runtime-id".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "no-such-runtime-id") .expect_err("configured default id matches nothing"); let message = error.to_string(); @@ -689,11 +728,7 @@ mod tests { let nightly = ready_runtime_manifest(&paths, "nightly-wheel-gfx94x-dcgpu-7-14-0")?; let manifests = [release, nightly]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -721,11 +756,7 @@ mod tests { ); let manifests = [ready, unusable]; - let config = RocmCliConfig { - default_runtime_id: Some("therock-release:gfx120X-all".to_owned()), - ..Default::default() - }; - let error = select_default_runtime(&config, &manifests) + let error = select_runtime_by_default_id(&manifests, "therock-release:gfx120X-all") .expect_err("two manifests share the configured default id"); let message = error.to_string(); @@ -847,7 +878,7 @@ mod tests { let unusable = unusable_runtime_manifest(&paths, "release-wheel-gfx94x-dcgpu-7-13-0")?; write_runtime_manifest(&paths, &unusable)?; - let error = select_runtime( + let error = persisting_select_runtime( &paths, &RocmCliConfig::default(), Some("release-wheel-gfx94x-dcgpu-7-13-0"), @@ -958,6 +989,305 @@ mod tests { Ok(()) } + /// Zero runtimes registered: the refusal points at the launcher's setup row, + /// and selecting wrote nothing. + #[test] + fn comfyui_selection_with_no_registered_runtime_says_to_install_rocm_and_writes_nothing() + -> Result<()> { + let paths = test_paths("comfyui-selection-none-registered"); + + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("no registered runtime means no selection") + .to_string(); + + assert!( + message.contains("Install ROCm first") + && message.contains("pick \"Set up this system\""), + "unexpected refusal: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + + /// One ready runtime and nothing chosen: it is the selection. + #[test] + fn comfyui_selection_with_one_ready_runtime_selects_it() -> Result<()> { + let paths = test_paths("comfyui-selection-one-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; + + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + Some(selected.python.display().to_string()), + manifest.python_executable + ); + Ok(()) + } + + /// Registered but none ready: the refusal lists the runtime with the reason + /// it is unusable, and fixing that reason (here: putting the interpreter + /// back) is what clears it. + #[test] + fn comfyui_selection_with_only_an_unusable_runtime_clears_once_it_is_fixed() -> Result<()> { + let paths = test_paths("comfyui-selection-none-ready"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = PathBuf::from( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + ); + fs::remove_file(&python)?; + + let message = persisting_select_runtime(&paths, &RocmCliConfig::default(), None) + .expect_err("an unusable runtime is not ready") + .to_string(); + + assert!( + message.contains("No ROCm runtime is ready") + && message.contains("rocm runtimes list") + && message.contains(&format!("`{}` (unusable", manifest.runtime_key)), + "unexpected refusal: {message}" + ); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + + fs::write(&python, "python")?; + let selected = persisting_select_runtime(&paths, &RocmCliConfig::default(), None)?; + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + Ok(()) + } + + /// Several ready runtimes and none chosen: the refusal names the remedies, + /// and each one clears it. `rocm runtimes activate ` is run as + /// `activate_runtime`, its implementation, and `--runtime-id` as the selector. + #[test] + fn comfyui_selection_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let paths = test_paths("comfyui-selection-several-ready"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + let second = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = persisting_select_runtime(&paths, &config, None) + .expect_err("several ready runtimes and no choice must not select") + .to_string(); + + assert!( + message.contains("Multiple ROCm runtimes are ready") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id ") + && message.contains(&first.runtime_key) + && message.contains(&second.runtime_key), + "unexpected refusal: {message}" + ); + + let selected = persisting_select_runtime(&paths, &config, Some(&first.runtime_key))?; + assert_eq!(selected.manifest.runtime_key, first.runtime_key); + + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &second.runtime_key)?; + let selected = persisting_select_runtime(&paths, &config, None)?; + assert_eq!(selected.manifest.runtime_key, second.runtime_key); + Ok(()) + } + + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// ComfyUI path, as it is by the engine paths, and finding it re-files its + /// registry entry, so the selection names a key `rocm runtimes list` shows. + #[test] + fn comfyui_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-recovery"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!( + config.setup.therock_venv.is_some(), + "setup must name the folder for this test to mean anything" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + + let selected = persisting_select_runtime(&paths, &config, None)?; + + assert_eq!(selected.manifest.runtime_key, manifest.runtime_key); + assert_eq!( + runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + Ok(()) + } + + /// Recovering the setup runtime can fail, and the ComfyUI path now reports + /// that failure as the engine paths do, rather than carrying on as if setup + /// had named nothing: a setup folder whose runtime is no longer usable is + /// named, with the reason, and nothing is written to the registry. + #[test] + fn comfyui_selection_reports_a_setup_runtime_that_is_no_longer_usable() -> Result<()> { + let paths = test_paths("comfyui-selection-setup-unusable"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let message = persisting_select_runtime(&paths, &config, None) + .expect_err("an unusable setup runtime is an error") + .to_string(); + + assert!( + message.contains("is not usable") && message.contains(&manifest.runtime_key), + "got: {message}" + ); + assert_eq!(runtime_registry_entry_names(&paths), Vec::::new()); + Ok(()) + } + + /// The two paths that pick "the one ready runtime" must not drift apart: + /// for each registry state below, ComfyUI selects a runtime exactly when + /// the engine paths do, and the same one. Both reach the rule through + /// `select_single_ready_runtime`, so a copy of it re-introduced in either + /// caller (a different readiness test, a missing setup recovery) shows up as + /// a disagreement on one of these states. + #[test] + fn comfyui_and_engine_paths_agree_on_the_single_ready_runtime() -> Result<()> { + use crate::engine_runtime_select::{ + resolve_engine_selection, validate_engine_selection_runtime, + }; + + #[derive(Clone, Copy)] + enum State { + Empty, + OneUnusable, + OneReady, + TwoReady, + ReadyAndUnusable, + SetupRuntimeLost, + SetupRuntimeLostBesideAReadyOne, + } + for (label, state, expected) in [ + ("empty", State::Empty, None), + ("one-unusable", State::OneUnusable, None), + ("one-ready", State::OneReady, Some("runtime-a")), + ("two-ready", State::TwoReady, None), + ( + "ready-and-unusable", + State::ReadyAndUnusable, + Some("runtime-a"), + ), + ( + "setup-runtime-lost", + State::SetupRuntimeLost, + Some("runtime-s"), + ), + ( + "setup-runtime-lost-beside-a-ready-one", + State::SetupRuntimeLostBesideAReadyOne, + None, + ), + ] { + // Each path gets its own copy of the state: a recovery run by one + // writes the registry, which would otherwise hand the other path a + // state it could not have found for itself. + let plant_state = |paths: &AppPaths| -> Result<()> { + let plant = |key: &str| { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + }; + let break_runtime = |manifest: &therock::InstalledRuntimeManifest| -> Result<()> { + fs::remove_file( + manifest + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + Ok(()) + }; + match state { + State::Empty => {} + State::OneUnusable => break_runtime(&plant("runtime-a")?)?, + State::OneReady => { + plant("runtime-a")?; + } + State::TwoReady => { + plant("runtime-a")?; + plant("runtime-b")?; + } + State::ReadyAndUnusable => { + plant("runtime-a")?; + break_runtime(&plant("runtime-b")?)?; + } + State::SetupRuntimeLost => { + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + State::SetupRuntimeLostBesideAReadyOne => { + plant("runtime-a")?; + crate::test_support::write_test_setup_runtime_missing_from_registry( + paths, + "runtime-s", + )?; + } + } + Ok(()) + }; + let comfyui_paths = test_paths(&format!("comfyui-engine-agree-{label}-comfyui")); + plant_state(&comfyui_paths)?; + let comfyui = persisting_select_runtime( + &comfyui_paths, + &RocmCliConfig::load(&comfyui_paths)?, + None, + ) + .ok() + .map(|selected| selected.manifest.runtime_key); + let engine_paths = test_paths(&format!("comfyui-engine-agree-{label}-engine")); + plant_state(&engine_paths)?; + let engine = validate_engine_selection_runtime( + &engine_paths, + resolve_engine_selection(&RocmCliConfig::load(&engine_paths)?, "vllm", None, None), + )? + .runtime_id; + + assert_eq!(comfyui, engine, "{label}: the two paths disagree"); + assert_eq!(comfyui.as_deref(), expected, "{label}"); + } + Ok(()) + } + fn command_env_value(command: &Command, key: &str) -> Option { command .get_envs() diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index e0c744891f..061f968b49 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -57,7 +57,10 @@ //! `resolve_runtime_selector_to_exact_key` is also `pub(crate)`: `engines_cmd.rs` //! calls it in production, and `runtimes_cmd.rs`'s own test module calls it //! directly too. `single_ready_runtime_key` is called only from -//! `validate_engine_selection_runtime` in this same module and stays private. +//! `validate_engine_selection_runtime` in this same module and stays private; +//! it does not decide which runtime counts as "the one ready runtime" itself, +//! it asks `runtime_adopt::select_single_ready_runtime`, the rule +//! `rocm comfyui install` shares. //! `optional_arg` and //! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read @@ -103,10 +106,8 @@ use rocm_core::{ }; use rocm_engine_protocol::EngineRecipeHint; -use crate::runtimes_cmd::{ - recover_setup_runtime_registration, select_runtime_manifest, - validate_runtime_manifest_for_activation, -}; +use crate::runtime_adopt::{SetupRecovery, SingleReadyRuntime, select_single_ready_runtime}; +use crate::runtimes_cmd::{recover_setup_runtime_registration, select_runtime_manifest}; use crate::therock; #[derive(Debug, Clone)] @@ -211,16 +212,14 @@ pub(crate) fn validate_engine_selection_runtime( fn single_ready_runtime_key(paths: &AppPaths) -> Result> { let config = RocmCliConfig::load(paths).unwrap_or_default(); - recover_setup_runtime_registration(paths, &config)?; - let manifests = therock::load_runtime_manifests(paths)?; - let ready = manifests - .iter() - .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) - .collect::>(); - Ok(match ready.as_slice() { - [manifest] => Some(manifest.runtime_key.clone()), - _ => None, - }) + // Zero or several ready runtimes leave the selection open: the caller that + // needs a runtime says so in its own words. + Ok( + match select_single_ready_runtime(paths, &config, SetupRecovery::Persist)? { + SingleReadyRuntime::One(manifest) => Some(manifest.runtime_key), + SingleReadyRuntime::Zero { .. } | SingleReadyRuntime::Many { .. } => None, + }, + ) } pub(crate) fn resolve_runtime_selector_to_exact_key( @@ -611,6 +610,7 @@ mod tests { use serde_json::json; use super::*; + use crate::runtime_registry::validate_runtime_manifest_for_activation; use crate::status_for_readiness; use crate::test_support::write_test_pip_runtime; use crate::tests::{test_app_paths, test_paths}; @@ -1048,4 +1048,95 @@ mod tests { let _ = std::fs::remove_dir_all(root); Ok(()) } + + /// Zero ready runtimes, the registry empty: the engine path answers with no + /// selection rather than an error, and writes nothing. The caller reports + /// the missing runtime in its own words (`engines_cmd.rs` and `serve_cmd.rs`). + #[test] + fn engine_selection_with_no_registered_runtime_stays_unselected_and_writes_nothing() + -> Result<()> { + let (root, paths) = test_paths("no-registered-runtime-selection"); + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.env_id.is_none()); + assert!(selection.source.is_none()); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new() + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Zero ready runtimes, one registered but unusable: still no selection. + /// Readiness is `validate_runtime_manifest_for_activation`, so a runtime + /// whose interpreter is gone does not count, and it is not picked as "the + /// only one" either. + #[test] + fn engine_selection_does_not_count_an_unusable_runtime_as_ready() -> Result<()> { + let (root, paths) = test_paths("unusable-runtime-selection"); + let manifest = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let python = manifest + .python_executable + .as_deref() + .expect("the fixture records an interpreter"); + std::fs::remove_file(python)?; + assert!(validate_runtime_manifest_for_activation(&manifest).is_err()); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert!(selection.runtime_id.is_none()); + assert!(selection.source.is_none()); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// The setup-recovery half of the rule: a ready runtime that only + /// `recover_setup_runtime_registration` can make visible is found by the + /// engine path, and finding it re-files its registry entry, so the message + /// and the state are one fact (the selection names a key that + /// `rocm runtimes list` now shows). + #[test] + fn engine_selection_recovers_the_setup_runtime_the_registry_lost() -> Result<()> { + let (root, paths) = test_paths("setup-recovery-selection"); + let manifest = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "release-pip-gfx120x-all-7-14-0", + )?; + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + Vec::::new(), + "the registry must start without the runtime for this test to mean anything" + ); + + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&RocmCliConfig::default(), "vllm", None, None), + )?; + + assert_eq!( + selection.runtime_id.as_deref(), + Some(manifest.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("single_ready_runtime")); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", manifest.runtime_key)] + ); + let _ = std::fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index 84301ee13f..eaee862b45 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -835,4 +835,72 @@ mod tests { ); } } + + /// Several ready runtimes and none chosen: the engine path does not guess. + /// `resolve_engine_env` reports the gap in its own words, and each remedy + /// those words name must clear it: `rocm runtimes activate ` + /// (run here as `activate_runtime`, its implementation) and `--runtime-id`. + #[test] + fn engine_install_with_several_ready_runtimes_names_remedies_that_clear_it() -> Result<()> { + let (root, paths) = test_paths("engine-install-several-ready-runtimes"); + let first = write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-13-0", + "therock-release:gfx120X-all", + "7.13.0", + 10, + )?; + write_test_pip_runtime( + &paths, + "release-pip-gfx120x-all-7-14-0", + "therock-release:gfx120X-all", + "7.14.0", + 20, + )?; + let mut config = RocmCliConfig::default(); + + let message = resolve_engine_env(&paths, &config, "vllm", None, None) + .expect_err("several ready runtimes and no choice must not resolve") + .to_string(); + assert!( + message.contains("no active ROCm runtime is configured") + && message.contains("rocm runtimes list") + && message.contains("rocm runtimes activate ") + && message.contains("--runtime-id"), + "unexpected refusal: {message}" + ); + + // `rocm runtimes activate `: the choice is now made. + crate::runtimes_cmd::activate_runtime(&paths, &mut config, &first.runtime_key)?; + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection(&config, "vllm", None, None), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!( + selection.source.as_deref(), + Some("config_active_runtime_key") + ); + + // `--runtime-id `: the same choice, for one invocation. + let selection = validate_engine_selection_runtime( + &paths, + resolve_engine_selection( + &RocmCliConfig::default(), + "vllm", + Some(&first.runtime_key), + None, + ), + )?; + assert_eq!( + selection.runtime_id.as_deref(), + Some(first.runtime_key.as_str()) + ); + assert_eq!(selection.source.as_deref(), Some("cli_runtime_id")); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/runtime_adopt.rs b/apps/rocm/src/runtime_adopt.rs index 549753818a..6c080accf5 100644 --- a/apps/rocm/src/runtime_adopt.rs +++ b/apps/rocm/src/runtime_adopt.rs @@ -5,9 +5,13 @@ //! Registering a runtime that was not installed by `rocm install sdk`: //! importing a manifest written elsewhere, adopting an existing Python //! environment by probing it, and recovering the registration of the runtime -//! `rocm setup` created. +//! `rocm setup` created. It also holds the one rule for picking "the one ready +//! runtime" when nothing else says which to use +//! ([`select_single_ready_runtime`]), which sits here because it recovers the +//! setup registration first; `rocm comfyui install` and the engine paths both +//! ask it and each words the zero and several cases itself. //! -//! Relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` +//! The rest was relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` //! calls into this module for `rocm runtimes import` and `rocm runtimes adopt`, //! and re-exports `newest_installed_runtime_manifest`, //! `recover_setup_runtime_registration` and `setup_runtime_paths` for the @@ -23,7 +27,7 @@ use rocm_core::{AppPaths, RocmCliConfig, managed_pip_cache_dir, runtime_python_e use crate::runtime_registry::{ paths_equivalent, runtime_manifest_path, runtime_registry_dir, - validate_runtime_manifest_for_activation, + sort_manifests_newest_install_first, validate_runtime_manifest_for_activation, }; use crate::therock; @@ -377,10 +381,17 @@ fn write_runtime_registry_manifest( Ok(()) } -pub(crate) fn recover_setup_runtime_registration( - paths: &AppPaths, - config: &RocmCliConfig, -) -> Result> { +/// The runtime `rocm setup` created, as the manifest in its install tree +/// describes it, and whether the runtime registry already holds its entry. +struct SetupRuntime { + manifest: therock::InstalledRuntimeManifest, + registered: bool, +} + +/// Find the runtime `rocm setup` created and check that it is usable, without +/// changing anything. `None` when setup named no folder or the folder holds no +/// manifest; an error when it does and the runtime cannot be used. +fn find_setup_runtime(paths: &AppPaths, config: &RocmCliConfig) -> Result> { let Some(setup_root) = config .setup .therock_venv @@ -424,16 +435,137 @@ pub(crate) fn recover_setup_runtime_registration( ) })?; - if !runtime_manifest_path(paths, &manifest.runtime_key).is_file() { - write_runtime_registry_manifest(paths, &manifest, false).with_context(|| { + let registered = runtime_manifest_path(paths, &manifest.runtime_key).is_file(); + Ok(Some(SetupRuntime { + manifest, + registered, + })) +} + +pub(crate) fn recover_setup_runtime_registration( + paths: &AppPaths, + config: &RocmCliConfig, +) -> Result> { + let Some(setup) = find_setup_runtime(paths, config)? else { + return Ok(None); + }; + if !setup.registered { + write_runtime_registry_manifest(paths, &setup.manifest, false).with_context(|| { format!( "failed to restore setup runtime `{}` into {}", - manifest.runtime_key, + setup.manifest.runtime_key, runtime_registry_dir(paths).display() ) })?; } - Ok(Some(manifest.runtime_key)) + Ok(Some(setup.manifest.runtime_key)) +} + +/// Whether finding the runtime `rocm setup` created may write its lost registry +/// entry back while a runtime is being picked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum SetupRecovery { + /// Write the entry back, as `rocm serve` and `rocm engines install` do. + Persist, + /// Count the setup runtime as if its entry were there, and write nothing: + /// a `--dry-run` preview must name the runtime the real run will use + /// without changing any file. + PreviewOnly, +} + +impl SetupRecovery { + /// The recovery a command with a `--dry-run` flag uses. + pub(crate) const fn for_dry_run(dry_run: bool) -> Self { + if dry_run { + Self::PreviewOnly + } else { + Self::Persist + } + } +} + +/// What picking "the one ready runtime" found. The caller decides how to report +/// the two cases that are not a pick, because the advice differs by command +/// (`rocm comfyui install` and `rocm serve` name different remedies). +#[derive(Debug)] +pub(crate) enum SingleReadyRuntime { + /// Exactly one registered runtime is ready. + One(Box), + /// No registered runtime is ready. `registered` is everything the registry + /// holds, unusable entries included and possibly none, so a caller can say + /// why each one does not count. + Zero { + registered: Vec, + }, + /// More than one is ready, so there is nothing to pick without a choice. + Many { + ready: Vec, + }, +} + +/// Pick the one ready runtime when nothing else says which to use: no +/// `--runtime-id`, no active runtime, no configured default. This is the only +/// place that rule lives; `rocm comfyui install` and the engine paths +/// (`rocm serve`, `rocm engines install`) both call it. +/// +/// Ready means `validate_runtime_manifest_for_activation` accepts the runtime, +/// the same check `rocm runtimes activate` applies. Setup is recovered first +/// (`recover_setup_runtime_registration`), so the runtime `rocm setup` created +/// counts even if its registry entry was lost. With [`SetupRecovery::Persist`] +/// that writes the entry back; with [`SetupRecovery::PreviewOnly`] nothing is +/// written, and the setup runtime is counted from its own manifest as the +/// registry would hold it once restored, so the answer is the one `Persist` +/// gives. +pub(crate) fn select_single_ready_runtime( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, +) -> Result { + match recovery { + SetupRecovery::Persist => { + recover_setup_runtime_registration(paths, config)?; + Ok(classify_single_ready_runtime( + therock::load_runtime_manifests(paths)?, + )) + } + SetupRecovery::PreviewOnly => { + let setup = find_setup_runtime(paths, config)?; + let mut registered = therock::load_runtime_manifests(paths)?; + if let Some(setup) = setup + && !setup.registered + { + // What `load_runtime_manifests` would return for the entry that + // `Persist` would have written. + registered.push( + setup + .manifest + .normalize_storage_paths() + .normalize_host_paths(), + ); + sort_manifests_newest_install_first(&mut registered); + } + Ok(classify_single_ready_runtime(registered)) + } + } +} + +/// The pure half of [`select_single_ready_runtime`]: which of `registered` are +/// ready, and whether that is none, one or several. +pub(crate) fn classify_single_ready_runtime( + registered: Vec, +) -> SingleReadyRuntime { + let ready: Vec<_> = registered + .iter() + .filter(|manifest| validate_runtime_manifest_for_activation(manifest).is_ok()) + .cloned() + .collect(); + if ready.is_empty() { + return SingleReadyRuntime::Zero { registered }; + } + match <[therock::InstalledRuntimeManifest; 1]>::try_from(ready) { + Ok([manifest]) => SingleReadyRuntime::One(Box::new(manifest)), + Err(ready) => SingleReadyRuntime::Many { ready }, + } } /// The folder setup was configured for and its pip cache — the pair both @@ -819,4 +951,220 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + fn pip_runtime(paths: &AppPaths, key: &str) -> Result { + write_test_pip_runtime(paths, key, "therock-release:gfx120X-all", "7.14.0", 20) + } + + /// Readiness is the activation check and nothing looser: a runtime whose + /// interpreter is gone is registered but does not count, and neither is it + /// "the only one". + #[test] + fn single_ready_runtime_counts_only_what_activation_accepts() -> Result<()> { + let (root, paths) = test_paths("single-ready-classify"); + let ready = pip_runtime(&paths, "runtime-a")?; + let broken = pip_runtime(&paths, "runtime-b")?; + fs::remove_file( + broken + .python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + assert!(validate_runtime_manifest_for_activation(&ready).is_ok()); + assert!(validate_runtime_manifest_for_activation(&broken).is_err()); + + match classify_single_ready_runtime(vec![ready.clone(), broken.clone()]) { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-a"), + other => panic!("expected the one ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(vec![broken.clone()]) { + SingleReadyRuntime::Zero { registered } => { + assert_eq!(registered.len(), 1, "unusable entries stay listed"); + } + other => panic!("expected no ready runtime, got {other:?}"), + } + match classify_single_ready_runtime(Vec::new()) { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + let second = pip_runtime(&paths, "runtime-c")?; + match classify_single_ready_runtime(vec![ready, broken, second]) { + SingleReadyRuntime::Many { ready } => { + let keys: Vec<_> = ready.iter().map(|m| m.runtime_key.as_str()).collect(); + assert_eq!(keys, ["runtime-a", "runtime-c"], "only ready ones listed"); + } + other => panic!("expected several ready runtimes, got {other:?}"), + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The selection recovers the setup runtime first, so a runtime the registry + /// lost is the one found, and finding it is what writes its entry back. + #[test] + fn select_single_ready_runtime_recovers_setup_before_counting() -> Result<()> { + let (root, paths) = test_paths("single-ready-recovers-setup"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + let config = RocmCliConfig::load(&paths)?; + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + match select_single_ready_runtime(&paths, &config, SetupRecovery::Persist)? { + SingleReadyRuntime::One(manifest) => assert_eq!(manifest.runtime_key, "runtime-s"), + other => panic!("expected the recovered setup runtime, got {other:?}"), + } + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + vec![format!("{}.json", lost.runtime_key)] + ); + + // Without setup naming a folder there is nothing to recover and the + // registry is not touched. + let (other_root, other_paths) = test_paths("single-ready-no-setup"); + match select_single_ready_runtime( + &other_paths, + &RocmCliConfig::default(), + SetupRecovery::Persist, + )? { + SingleReadyRuntime::Zero { registered } => assert!(registered.is_empty()), + other => panic!("expected no ready runtime, got {other:?}"), + } + assert!(crate::test_support::runtime_registry_entry_names(&other_paths).is_empty()); + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(other_root); + Ok(()) + } + + /// The keys `select_single_ready_runtime` found ready, in the order it lists + /// them, or the message of its refusal to recover. + fn single_ready_outcome( + paths: &AppPaths, + config: &RocmCliConfig, + recovery: SetupRecovery, + ) -> Result, String> { + match select_single_ready_runtime(paths, config, recovery) { + Ok(SingleReadyRuntime::One(manifest)) => Ok(vec![manifest.runtime_key]), + Ok(SingleReadyRuntime::Many { ready }) => { + Ok(ready.into_iter().map(|m| m.runtime_key).collect()) + } + Ok(SingleReadyRuntime::Zero { registered }) => { + Err(format!("none ready of {}", registered.len())) + } + Err(error) => Err(format!("{error:#}")), + } + } + + /// A preview counts the runtime setup created from its own manifest and + /// writes nothing; the real run then finds the same runtime and writes the + /// entry. Both orders of install time are covered: the ready runtimes are + /// listed newest first, and a preview must list them the way the registry + /// will once the entry is back. + #[test] + fn preview_counts_the_setup_runtime_without_writing_and_agrees_with_persist() -> Result<()> { + for (label, other_installed_at) in [("other-newer", 30), ("other-older", 10)] { + let (root, paths) = test_paths(&format!("single-ready-preview-{label}")); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-setup", + )?; + write_test_pip_runtime( + &paths, + "runtime-other", + "therock-release:gfx120X-all", + "7.14.0", + other_installed_at, + )?; + let config = RocmCliConfig::load(&paths)?; + let registry_before = crate::test_support::runtime_registry_entry_names(&paths); + assert_eq!(registry_before, ["runtime-other.json"], "{label}"); + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect("the setup runtime and the other one are both ready"); + + assert_eq!(preview.len(), 2, "{label}: {preview:?}"); + assert!(preview.contains(&lost.runtime_key), "{label}: {preview:?}"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + registry_before, + "{label}: a preview must not write the lost entry" + ); + let persisted = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect("recovery finds the same two"); + assert_eq!(preview, persisted, "{label}: preview and real run disagree"); + assert_eq!( + crate::test_support::runtime_registry_entry_names(&paths), + [ + "runtime-other.json".to_owned(), + "runtime-setup.json".to_owned() + ], + "{label}: the real run writes the entry back" + ); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + /// The setup runtime alone is "the one ready runtime" in a preview, and a + /// registry that already holds it is left as it is. + #[test] + fn preview_picks_a_lost_setup_runtime_and_ignores_a_registered_one() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-one"); + crate::test_support::write_test_setup_runtime_missing_from_registry(&paths, "runtime-s")?; + let config = RocmCliConfig::load(&paths)?; + + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + + // Once the entry exists a preview and a real run see one runtime, and + // neither adds a second copy of it. + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::Persist), + Ok(vec!["runtime-s".to_owned()]) + ); + assert_eq!( + single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly), + Ok(vec!["runtime-s".to_owned()]) + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A setup runtime that is no longer usable is reported by a preview as the + /// real run reports it, and neither writes anything. + #[test] + fn preview_reports_an_unusable_setup_runtime_as_persist_does() -> Result<()> { + let (root, paths) = test_paths("single-ready-preview-unusable"); + let lost = crate::test_support::write_test_setup_runtime_missing_from_registry( + &paths, + "runtime-s", + )?; + fs::remove_file( + lost.python_executable + .as_deref() + .context("the fixture records an interpreter")?, + )?; + let config = RocmCliConfig::load(&paths)?; + + let preview = single_ready_outcome(&paths, &config, SetupRecovery::PreviewOnly) + .expect_err("an unusable setup runtime is an error"); + let persist = single_ready_outcome(&paths, &config, SetupRecovery::Persist) + .expect_err("an unusable setup runtime is an error"); + + assert!(preview.contains("is not usable"), "{preview}"); + assert_eq!(preview, persist); + assert!(crate::test_support::runtime_registry_entry_names(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn a_dry_run_previews_and_a_real_run_persists() { + assert_eq!(SetupRecovery::for_dry_run(true), SetupRecovery::PreviewOnly); + assert_eq!(SetupRecovery::for_dry_run(false), SetupRecovery::Persist); + } } diff --git a/apps/rocm/src/runtime_registry.rs b/apps/rocm/src/runtime_registry.rs index 9f9fa17ce7..5e0ce19d02 100644 --- a/apps/rocm/src/runtime_registry.rs +++ b/apps/rocm/src/runtime_registry.rs @@ -121,7 +121,7 @@ impl InstalledRuntimeManifest { wheel_composition_includes_devel(self.wheel_composition.as_ref()).unwrap_or(self.devel) } - fn normalize_host_paths(mut self) -> Self { + pub(crate) fn normalize_host_paths(mut self) -> Self { self.install_root = normalize_manifest_path(self.install_root); self.python_launcher = self .python_launcher diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index c724ad973f..9cbb22aa61 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -229,6 +229,43 @@ pub(crate) fn write_test_pip_runtime( Ok(manifest) } +/// Leave `paths` as `rocm setup` followed by a lost registry entry would: a ready +/// wheel runtime whose install tree holds its own `.rocm-cli-runtime.json` and +/// that `config.json` names as the setup folder, but that the runtime registry +/// has no entry for. Only `recover_setup_runtime_registration` can make it +/// visible again, so a selection that finds it has run that recovery. +pub(crate) fn write_test_setup_runtime_missing_from_registry( + paths: &AppPaths, + runtime_key: &str, +) -> Result { + let manifest = write_test_pip_runtime( + paths, + runtime_key, + "therock-release:gfx120X-all", + "7.13.0", + 20, + )?; + fs::remove_file(runtime_manifest_path(paths, runtime_key))?; + let mut config = rocm_core::RocmCliConfig::default(); + config.setup.therock_venv = Some(manifest.install_root.clone()); + config.save(paths)?; + Ok(manifest) +} + +/// File names in the runtime registry directory, sorted; empty when the +/// directory does not exist. Lets a test assert that a selection did, or did +/// not, write a registry entry. +pub(crate) fn runtime_registry_entry_names(paths: &AppPaths) -> Vec { + let mut names: Vec = fs::read_dir(runtime_registry_dir(paths)) + .into_iter() + .flatten() + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + mod tests { use super::*; diff --git a/docs/architecture.md b/docs/architecture.md index 724bc81c0f..4ec3b98f78 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -50,7 +50,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. - [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it outside its own tests. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. One test here, `linux_only_engine_runtime_status_is_explicit_on_native_windows`, also calls `model_registry_adapter_availability_note` from `model_catalog_text.rs`, which is `pub(crate)` for that reason; its primary subject is `engine_runtime_status_label`, so it stays. @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`providers/local_http.rs`](../apps/rocm/src/providers/local_http.rs) — full domain extraction, a child of `providers.rs`: the local provider's hand-rolled HTTP/1.1 transport. `post_json_to_local_endpoint` (with `post_json_to_local_endpoint_body` and `local_bearer_header`) is the blocking JSON POST; `stream_json_from_local_endpoint` is the streamed POST, which reads the response in whichever body framing the engine answers with (`read_sse_body` picks between `read_chunked_sse_body`, `read_fixed_sse_body` and `read_close_delimited_sse_body` from the headers, via `header_value`) and feeds an `OpenAiSseEmitter`; `body_truncated_at_eof` and `LOCAL_STREAM_TRUNCATED` name the failure when an engine closes the connection mid-answer; `parse_http_endpoint` reads the `http://` endpoint URL. `post_json_to_local_endpoint` and `stream_json_from_local_endpoint` are `pub(super)`, called from `LocalProvider`, and `LOCAL_STREAM_TRUNCATED` is also `pub(super)` (imported by `providers.rs` for its test module only); the rest is private. Its edges back into `providers` are `OpenAiSseEmitter` from `providers/wire.rs`, the `SseBytesEmitter` trait the emitter is pushed bytes through (which stays in `providers.rs`), and the shared `ProviderStreamEvent` type; nothing in `wire.rs` reads `local_http.rs`, so the edge is one-way. [`providers/local_http/proptests.rs`](../apps/rocm/src/providers/local_http/proptests.rs) holds the property tests of the SSE body readers and is declared from this module so they sit with the readers they test; the example-based test of `parse_http_endpoint` is in this module's own `mod tests`, and the tests that drive `LocalProvider` end to end against a mock server stay in `providers.rs`'s `mod tests` with their server helpers. - [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) — full domain extraction, a child of `providers.rs`: the OpenAI-compatible and Anthropic wire formats. The request-body builders (`openai_chat_request_body`, `openai_stream_chat_request_body`, `openai_local_chat_request_body`, `openai_local_stream_chat_request_body`, `anthropic_chat_request_body`, `anthropic_stream_chat_request_body`, and the helpers behind them, including `local_openai_compatible_request`, `apply_local_model_request_defaults` and `local_model_should_disable_thinking`), the parsers for non-streaming responses (`parse_openai_chat_message`, `parse_anthropic_chat_message`), and the SSE emitters with their line parsers (`OpenAiSseEmitter` and `AnthropicSseEmitter`, which implement the `SseBytesEmitter` trait that stays in `providers.rs`; `parse_openai_sse_line`, `parse_anthropic_sse_line`). The builders, the two chat-message parsers and the two emitters (with their `new` and `finish`) are `pub(super)`, called from `LocalProvider` and `RemoteProvider`; `OpenAiSseEmitter` is also read by `providers/local_http.rs`. Depends on the shared request/response types in `providers.rs` and on `providers/tool_schema.rs`, and on nothing else. The tests of the builders and parsers live in this module's own `mod tests`, with the tool-catalog assertions that read through the request bodies, and [`providers/wire/sse_emitter_proptests.rs`](../apps/rocm/src/providers/wire/sse_emitter_proptests.rs), the split-invariance and panic-guard properties of the emitters and line parsers, is declared from this module; the stream tests that run an emitter against a mock server stay in `providers.rs`'s `mod tests`. - [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs) — the argv helpers that `chat.rs`, `tool_policy.rs` and the freeform planner share, in a leaf module whose only `crate::` dependency is the leaf [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs): `chat_cli_arg_value`, `chat_cli_arg_value_checked`, `chat_cli_has_flag` and `ensure_flag` (from `chat.rs`), and `format_structured_tool_call` (which quotes each argument with `shell_quote`), `serve_args_request_cpu_device` and `device_policy_is_cpu_mode` (from `freeform_planner.rs`). They moved so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `ensure_flag` (read by `freeform_execution.rs`) and `freeform_planner.rs` re-exports `format_structured_tool_call`, so every other caller keeps its path. -- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. +- [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): registering a runtime that `rocm install sdk` did not install. `import_runtime_manifest` records a manifest written elsewhere, `adopt_runtime_from_python_options` (with `AdoptRuntimeOptions`, `AdoptRuntimeRequest`, `infer_adopt_runtime_request` and `adopt_runtime_from_probe`) probes an existing Python environment and registers it, and `recover_setup_runtime_registration` registers the runtime `rocm setup` created, with `newest_installed_runtime_manifest` and `setup_runtime_paths` beside it. `select_single_ready_runtime` is the one rule for picking "the one ready runtime" when nothing else says which to use: it recovers the setup runtime first, then returns a `SingleReadyRuntime` (`One`, `Zero` with every registered runtime, or `Many` with the ready ones), where ready is `validate_runtime_manifest_for_activation`; its pure half is `classify_single_ready_runtime`. Its `SetupRecovery` argument says whether recovering the setup runtime may write its lost registry entry back (`Persist`, what the engine paths and a real ComfyUI install use) or only count the runtime from its own manifest and write nothing (`PreviewOnly`, for a `--dry-run` preview, which must name the runtime the real run will use without changing a file); `SetupRecovery::for_dry_run` maps the flag. Both [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) (which treats `Zero` and `Many` as no selection and leaves the advice to its callers) and [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) (which words them as refusals) call it directly rather than through `runtimes_cmd.rs`, and neither keeps its own copy. It depends on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (manifest paths, activation validation, `paths_equivalent`) and `therock.rs`, not on `runtimes_cmd.rs`, which calls into it and re-exports `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths`. `import_runtime_manifest`, `AdoptRuntimeOptions`, `adopt_runtime_from_python_options` (with the fields of `AdoptRuntimeOptions` and `AdoptRuntimeRequest`) and `adopt_runtime_from_probe` are `pub(crate)` for `runtimes()` and for the `runtimes_cmd.rs` tests that adopt or import a runtime and then list or activate it; `newest_installed_runtime_manifest`, `recover_setup_runtime_registration` and `setup_runtime_paths` are `pub(crate)` too, for the modules that import them through `runtimes_cmd.rs`'s re-exports; the rest is private. Its tests cover the import and adopt rules that do not involve activation or listing. - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install: the orchestration (`finish_sdk_install`, `settle_engine_install`, `finalize_successful_sdk_install`), the engine-dependency check, the runtime device check and the post-install verdict built on it, and `SdkInstallConsents`. The torch alignment and retention it runs live in the child module [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs). `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`; `settle_engine_install` is `pub(crate)` and imported by [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), so the import edge between the two runs one way, from `engines_cmd.rs` to this module. This module (and its torch-alignment child) does not import `engines_cmd.rs`: the engine environment and runtime-selection helpers it needs (`engine_manages_own_runtime`, `env_root_for_engine_install`, `runtime_key_for_python`, and `runtime_manifest_for_selector` from the torch-alignment child) are imported from [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs). The host system-package installs live in [`system_packages.rs`](../apps/rocm/src/system_packages.rs); this module re-exports `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` (called from both `install_cmd.rs`'s `install()` and `engines_cmd.rs`) and `ensure_openmpi_for_vllm` (called from `engines_cmd.rs` and, for the engine auto-install after an SDK install, from here) so those callers keep their imports. - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. @@ -115,7 +115,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: - [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) — test-only (`#[cfg(test)]`): checks that every `rocm …`/`rocmd …` command the repository advises — in `--help` text, production string literals, the markdown docs and skills, and the VHS tapes — exists and parses, by running it through the parser of the binary it names (`rocm`'s clap `Cli`, or `rocmd`'s `parse_args`). - [`comfyui/install.rs`](../apps/rocm/src/comfyui/install.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): `rocm comfyui install`, including its dry run and the reinstall that replaces ComfyUI's code (through [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs)) while refusing to do so under a running ComfyUI. It downloads and unpacks the source archive, installs ComfyUI's Python dependencies into the selected runtime with uv (filtering out and pinning the runtime's own torch stack, and streaming uv's output to the install log) and runs the AMD GPU check. The manifest, the install log paths and the GPU probe stay in `comfyui.rs`. It imports `runtime_env` and, for the running-ComfyUI guard, `process`; neither imports it. Its unit tests and the Unix-only `archive_properties` property tests for the source unpack live in the module; the fixtures they share with the tests left in `comfyui.rs` stay there. - [`comfyui/process.rs`](../apps/rocm/src/comfyui/process.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): the lifecycle of the ComfyUI process this CLI launches. `start` checks the installed folder, runs the GPU check, launches ComfyUI in the background (a detached child on Linux, a hidden PowerShell launcher on Windows) and waits for its port; `stop` ends the saved process tree. `evaluate_running_state` tells a running, a still-loading and a stopped ComfyUI apart for `status`, and `running_from`, `refuse_while_running` and `running_advice` are the guard that refuses a reinstall while ComfyUI runs from the folder it would replace. The saved state and the types in it stay in `comfyui.rs`. Among the `comfyui` children it imports from [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) and uses [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) to check for an interrupted reinstall; neither imports it, so there is no cycle. -- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. +- [`comfyui/runtime_env.rs`](../apps/rocm/src/comfyui/runtime_env.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): choosing which installed ROCm runtime ComfyUI runs against (`select_runtime`, with the messages that tell the user how to pick one when none or several are ready) and building the `VIRTUAL_ENV`, `ROCM_PATH`, `PATH` and library-path environment a ComfyUI child process runs with. When no `--runtime-id`, active runtime or configured default says which runtime to use, `select_runtime` does not decide which runtimes count as ready itself: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, the same rule the engine paths use, and only words the answer (`single_ready_runtime_or_refusal`), so ComfyUI keeps its own remediation text. `select_runtime` takes the `SetupRecovery` to ask it with, which the install derives from `--dry-run`, so a preview does not write the registry entry the real run would restore. The types it returns (`SelectedRuntime`, `ComfyUiRuntimeEnvironment`) stay in `comfyui.rs`, which the module imports from; it imports nothing from the other `comfyui` children. - [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs) — child module of [`comfyui.rs`](../apps/rocm/src/comfyui.rs): replacing ComfyUI's code in an existing `source/` folder on reinstall without touching anything else there. - [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) — `rocm remote`: driving a remote GPU host from this machine, with SSH as the control channel and the remote's own service published onto the tailnet for inference traffic. Like [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs), it owns its command enum (`RemoteCommand`) and dispatch fn (`run`). Its `pub(crate)` submodules: - [`remote/bootstrap.rs`](../apps/rocm/src/remote/bootstrap.rs) — deciding whether a remote machine can host a model before asking it to. diff --git a/docs/testing.md b/docs/testing.md index eb9f36502e..f30b66650f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -780,6 +780,18 @@ cargo xtask e2e -- -n "comfyui-(0[5-9]|1[0-3])" `-n` bypasses the suite's applicability resolver, so the `@requires-os:linux` gate is not applied: run it only on Linux. +`comfyui-15` and `comfyui-16` cover which runtime `rocm comfyui install` uses +when none is chosen: it recovers the runtime `rocm setup` created before +counting the ready ones, as `rocm serve` does, so a runtime whose registry entry +was lost is found. The real run (`comfyui-16`, against an existing checkout that +needs no dependencies) writes the entry back; the `--dry-run` preview +(`comfyui-15`) names the same runtime but writes nothing, because a preview +promises not to change files. Both run with no GPU: + +```bash +cargo xtask e2e -- -n "comfyui-1[56]" +``` + `comfyui-14` is different: it installs ComfyUI into a real, isolated managed runtime and checks the ROCm torch stack survives with no `nvidia-*` packages added. It carries `@requires-gpu @nightly @requires-os:linux @requires-bare-metal` diff --git a/tests/e2e-cucumber/features/comfyui.feature b/tests/e2e-cucumber/features/comfyui.feature index 036d9615a5..9aee470019 100644 --- a/tests/e2e-cucumber/features/comfyui.feature +++ b/tests/e2e-cucumber/features/comfyui.feature @@ -328,3 +328,34 @@ Feature: ComfyUI install reports progress, makes failures actionable and preserv And ComfyUI's dependencies were installed into the runtime And the runtime's torch stack is unchanged And no nvidia-* packages were added to the runtime + + # With no runtime chosen, `rocm comfyui install` uses the one ready runtime, + # and it finds it the way `rocm serve` and `rocm engines install` do: it + # recovers the runtime `rocm setup` created first. A runtime whose registry + # entry was lost, but whose install tree still holds its own manifest and that + # setup names, therefore counts as the ready runtime, instead of the install + # refusing with "Install ROCm first" while setup has finished. The real run + # writes the lost registry entry back (comfyui-16); a `--dry-run` preview + # names the same runtime but leaves the registry as it found it, because it + # promises not to change files (this scenario). The preview prints the + # `--runtime-id` of the command it would run, which is how it shows the + # choice. No GPU is needed: readiness is filesystem and manifest state. + @id:comfyui-install-finds-the-runtime-setup-created + Scenario: comfyui-15 - A ComfyUI install preview finds the runtime setup created without writing the registry + Given a ready ROCm runtime that setup created and the registry has forgotten + When the user previews a ComfyUI install without choosing a runtime + Then the preview selects that runtime + And the runtime registry still has no entry for it + + # The real run of the same install: it uses the runtime setup created and + # writes its lost registry entry back, so the runtime it installed into is one + # `rocm runtimes list` shows afterwards. The ComfyUI checkout is already in + # place and asks for nothing beyond the torch stack the runtime provides, so + # the install needs no network and no package index. + @id:comfyui-install-restores-the-runtime-setup-created + Scenario: comfyui-16 - A ComfyUI install restores the registry entry of the runtime setup created + Given a ready ROCm runtime that setup created and the registry has forgotten + And a ComfyUI checkout that needs nothing beyond the runtime's torch stack + When the user installs ComfyUI without choosing a runtime + Then the install uses the runtime setup created + And the runtime registry has an entry for it again diff --git a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs index 7fe33a9917..98f75637aa 100644 --- a/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/comfyui_steps.rs @@ -34,6 +34,15 @@ //! (`crates/rocm-dash-tui/src/app/tool_approval.rs`) asserts that envelope is collapsed //! out of the chat. //! +//! `comfyui-15` and `comfyui-16` cover the other half of that selection: with +//! nothing chosen, the install recovers the runtime `rocm setup` created before +//! it counts the ready ones, as `rocm serve` does, so a runtime the registry +//! lost is found. They plant a ready runtime and the setup folder that names it, +//! but no registry entry. `comfyui-15` previews the install with `--dry-run`, +//! which prints the `--runtime-id` of the runtime it chose and writes nothing, +//! so the registry still has no entry afterwards; `comfyui-16` runs the install +//! against an existing checkout and finds the entry written back. +//! //! `comfyui-04` covers the source-archive download spinner under a real PTY, //! mirroring `download_progress_pty.feature`'s tarball scenario but for //! `download_and_extract_source`'s in-process `GzDecoder`/`tar` unpack, which @@ -194,6 +203,21 @@ fn rocm_library_filename(shortname: &str) -> String { /// confusing downstream assertion instead of on the fixture. If this step starts /// failing after an edit here, suspect the manifest shape first. fn plant_ready_runtime(data: &Path, key: &str) { + let manifest = plant_ready_runtime_tree(data, key); + let registry = data.join("runtimes").join("registry"); + std::fs::create_dir_all(®istry) + .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); + std::fs::write( + registry.join(format!("{key}.json")), + serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), + ) + .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); +} + +/// The on-disk half of [`plant_ready_runtime`]: the install root and what the +/// readiness check needs under it. Returns the manifest JSON that describes it, +/// leaving it to the caller whether the registry gets an entry. +fn plant_ready_runtime_tree(data: &Path, key: &str) -> serde_json::Value { let install_root = data.join("runtimes").join("roots").join(key); let sdk_root = install_root.join("sdk"); let sdk_bin = sdk_root.join("bin"); @@ -208,7 +232,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { write_fixture(&amdhip, "stub"); write_fixture(&hipblas, "stub"); - let manifest = serde_json::json!({ + serde_json::json!({ "runtime_key": key, "runtime_id": key, "channel": "release", @@ -229,16 +253,7 @@ fn plant_ready_runtime(data: &Path, key: &str) { ], }, "installed_at_unix_ms": 1, - }); - - let registry = data.join("runtimes").join("registry"); - std::fs::create_dir_all(®istry) - .unwrap_or_else(|e| panic!("failed to create {}: {e}", registry.display())); - std::fs::write( - registry.join(format!("{key}.json")), - serde_json::to_vec_pretty(&manifest).expect("manifest serialises"), - ) - .unwrap_or_else(|e| panic!("failed to write the planted runtime manifest: {e}")); + }) } /// Combined stdout+stderr of the recorded `rocm` invocation. The refusal is an @@ -536,6 +551,126 @@ async fn refusal_lists_both_keys(world: &mut E2eWorld) { } } +// --- comfyui-15: the install finds the runtime setup created --- + +/// The one runtime `comfyui-15` plants. Not in [`RUNTIME_KEYS`], which are the +/// two the ambiguity scenario needs. +const SETUP_RUNTIME_KEY: &str = "e2e-setup-created-runtime"; + +fn registry_dir(world: &E2eWorld) -> PathBuf { + data_dir(world).join("runtimes").join("registry") +} + +#[given("a ready ROCm runtime that setup created and the registry has forgotten")] +async fn plant_runtime_setup_created_and_registry_forgot(world: &mut E2eWorld) { + let data = data_dir(world); + let manifest = plant_ready_runtime_tree(&data, SETUP_RUNTIME_KEY); + let install_root = data.join("runtimes").join("roots").join(SETUP_RUNTIME_KEY); + // The install tree's own copy of its manifest is what makes the registry + // entry recoverable. Without it recovery finds nothing to restore and the + // scenario would pass or fail for the wrong reason. + write_fixture( + &install_root.join(".rocm-cli-runtime.json"), + &serde_json::to_string_pretty(&manifest).expect("manifest serialises"), + ); + // Black-box: plain JSON matching the CLI's on-disk config schema. `setup` + // naming the folder is what lets recovery find the tree; there is no + // registry entry, and that is the state under test. + write_fixture( + &root(world).join("config").join("config.json"), + &serde_json::to_string_pretty(&serde_json::json!({ + "setup": { "therock_venv": install_root.display().to_string() }, + })) + .expect("config serialises"), + ); + assert!( + !registry_dir(world) + .join(format!("{SETUP_RUNTIME_KEY}.json")) + .exists(), + "the registry must start without the runtime for this scenario to mean anything" + ); +} + +#[when("the user previews a ComfyUI install without choosing a runtime")] +async fn preview_comfyui_install_without_runtime(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["comfyui", "install", "--dry-run"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the preview selects that runtime")] +async fn preview_selects_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the preview should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.lines().any(|line| line.contains("install command:") + && line.contains(&format!("--runtime-id {SETUP_RUNTIME_KEY}"))), + "the preview should name the runtime setup created in its install command, got:\n{text}" + ); +} + +#[then("the runtime registry has an entry for it again")] +async fn registry_has_the_setup_runtime_again(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + entry.is_file(), + "selecting the runtime setup created should have written {} back", + entry.display() + ); +} + +/// A preview promises not to change files, and the registry entry is a file: the +/// runtime it named is not registered by having been previewed. +#[then("the runtime registry still has no entry for it")] +async fn registry_still_lacks_the_setup_runtime(world: &mut E2eWorld) { + let entry = registry_dir(world).join(format!("{SETUP_RUNTIME_KEY}.json")); + assert!( + !entry.exists(), + "a `--dry-run` preview must not write {}", + entry.display() + ); +} + +#[given("a ComfyUI checkout that needs nothing beyond the runtime's torch stack")] +async fn comfyui_checkout_needing_no_dependencies(world: &mut E2eWorld) { + let source_dir = data_dir(world) + .join("runtimes") + .join("roots") + .join(SETUP_RUNTIME_KEY) + .join("apps") + .join("comfyui") + .join("source"); + // Every ComfyUI release has a `main.py`. The requirements name only the + // torch stack the runtime provides, which the install filters out, so it + // reaches no package index. + write_fixture(&source_dir.join("main.py"), ""); + write_fixture( + &source_dir.join("requirements.txt"), + "torch==2.4.0\ntorchvision==0.19.0\ntorchaudio==2.4.0\n", + ); +} + +#[then("the install uses the runtime setup created")] +async fn install_uses_the_setup_runtime(world: &mut E2eWorld) { + let text = refusal_text(world); + assert_eq!( + world.cli_rc, + Some(0), + "the install should succeed, got:\n{text}" + ); + let stdout = world.cli_output.clone().unwrap_or_default(); + assert!( + stdout.contains("action: install") && stdout.contains("ROCm runtime: 7.13.0"), + "the install should report the runtime setup created, got:\n{text}" + ); +} + /// Pacing knobs for `comfyui-04`'s download server, mirroring /// `therock_steps.rs`'s `PACED_TARBALL_*` constants: large enough that several /// chunk boundaries land before the transfer completes, slow enough per chunk