diff --git a/README.md b/README.md index 22cb10f496..7c4c85abfd 100644 --- a/README.md +++ b/README.md @@ -754,6 +754,28 @@ being followed. The two archive rows the report prints are tagged `note: can be downloaded again; safe to remove`, so it is visible which rows `remove-downloads` acts on. +The report also lists `Engine installs`: each Lemonade copy on disk and its +size. Lemonade installs into the active ROCm runtime's own folder when one is +active and into the data folder otherwise, so a machine that installed +Lemonade before its first SDK can hold two full copies (about 25 GiB each). The +copy Lemonade runs from is `in use`. A copy it is not using right now is marked +`not used right now`, which is not a promise it stays unused: Lemonade installs +into the active ROCm runtime's folder, so a copy inside a ROCm install is reused +as soon as that install becomes active, and is never marked reclaimable. Only +the data folder's copy, which no ROCm install owns, is marked reclaimable, and +nothing removes it automatically: delete its `runtime`, `downloads` and `extract` +folders to free the space. A copy is only called unused when Lemonade's install +record points at another copy that is really there: if the record names a folder +with no install, or cannot be read, the report says so and marks no copy +reclaimable. A copy inside a runtime folder +also counts toward that runtime's size. `rocm runtimes uninstall` removes it +with the runtime and clears Lemonade's install record, so Lemonade is no longer +recorded as installed; it says so when the record could not be cleared, and when +it could not be read it names `rocm engines install lemonade`, which rewrites +the record and keeps a current install. +For an external or imported runtime, uninstall only unregisters it, so its +Lemonade copy stays and the report does not say otherwise. + The report also lists `local server records`: one JSON record plus the engine's log for each `rocm serve --managed` launch, kept after the server stops. `rocm services list --all` lists them, and no `rocm storage` command removes @@ -783,6 +805,18 @@ and installs once more; see [Lemonade backend alignment](https://github.com/ROCm/rocm-cli/blob/main/docs/engine-plugins.md#lemonade-backend-alignment-on-engine-install) for the details and for `ROCM_CLI_DISABLE_LEMONADE_BACKEND_ALIGNMENT`, which turns both off. +Before Lemonade writes a new install, whether from `rocm engines install +lemonade` or the first `rocm serve` that needs it, it prints the folder it is +installing into, the download (up to about 4.4 GiB) and disk use (up to about +25 GiB) of its llama.cpp ROCm backend, and any Lemonade install it is not +reusing. Those are upper bounds: Lemonade picks the backend after the install +starts, and a smaller one needs less. So it stops before downloading only when +the drive cannot hold Lemonade itself (about 16 MiB), and with less room than +the ROCm backend or the finished install it warns and continues. An install that +replaces the existing one (`--reinstall`, or a new Lemonade version) deletes it +first, so its space counts as free and the notice says so. On Windows the +backend download's own progress appears in the console. + `--yes` approves the system-package installs vLLM needs (OpenMPI, libatomic and libnuma), including a `sudo` password prompt. `engines install` never asks for confirmation. Without `--yes`, a Linux host with root or passwordless `sudo` diff --git a/apps/rocm/src/runtime_uninstall.rs b/apps/rocm/src/runtime_uninstall.rs index 8b102383e6..e140cf1550 100644 --- a/apps/rocm/src/runtime_uninstall.rs +++ b/apps/rocm/src/runtime_uninstall.rs @@ -5,12 +5,16 @@ //! `rocm runtimes uninstall`: planning the removal of a registered runtime, //! re-validating that plan after the confirmation prompt, and applying it //! (unregistering the runtime, clearing the active and default state that named -//! it, and removing its install root when ROCm CLI owns it). +//! it, and removing its install root when ROCm CLI owns it). When that root +//! holds the Lemonade install its record points at (a self-managed engine +//! installs inside the active runtime's folder), the plan says so and applying +//! it clears the record, so Lemonade does not keep naming deleted files. //! -//! Relocated from `runtimes_cmd.rs` with no behavior change. `runtimes_cmd.rs` -//! calls into this module from `runtimes()` and re-exports `uninstall_runtime`, -//! `should_remove_runtime_install_root` and `InstallRootDecision` for -//! `storage.rs`. The active-runtime marker check this module makes +//! Relocated from `runtimes_cmd.rs`; the Lemonade record handling is new. +//! `runtimes_cmd.rs` calls into this module from `runtimes()` and re-exports +//! `uninstall_runtime`, `should_remove_runtime_install_root`, +//! `InstallRootDecision` and `lemonade_record_report_line` for `storage.rs`. +//! The active-runtime marker check this module makes //! (`active_runtime_marker_matches`, over the `ActiveRuntimeMarker` the //! activation writes) and the path comparison it needs live in //! `runtime_registry.rs`, so this module does not depend on `runtimes_cmd.rs`. @@ -20,6 +24,7 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, RocmCliConfig, runtime_install_root_is_protected}; +use rocm_engine_lemonade::RecordCleanup; use crate::cli_report; use crate::runtime_registry::{ @@ -34,6 +39,9 @@ pub(crate) struct RuntimeUninstallResult { pub(crate) runtime_key: String, pub(crate) registry_path: PathBuf, pub(crate) removed_install_root: Option, + /// What happened to Lemonade's install record once the folder was removed: + /// cleared, left dangling because deleting it failed, or not checkable. + pub(crate) lemonade_record: RecordCleanup, pub(crate) read_only: bool, pub(crate) manifest_mismatch: bool, pub(crate) was_active: bool, @@ -70,6 +78,14 @@ pub(crate) struct RuntimeUninstallPlan { /// does not by itself orphan the default while others remain). clears_default_runtime: bool, install_root_decision: InstallRootDecision, + /// Lemonade's runtime folder when its install record points inside this + /// runtime's folder: a self-managed engine installs there while the runtime + /// is active, so removing the folder removes that engine too. + lemonade_install_inside: Option, + /// Whether this runtime's folder holds a Lemonade tree at all, installed or + /// partly installed. Only then can an unreadable install record matter to + /// this uninstall. + holds_lemonade_tree: bool, } impl RuntimeUninstallPlan { @@ -78,6 +94,40 @@ impl RuntimeUninstallPlan { } } +/// The line `runtimes uninstall` (and `storage remove-old-installs`, which +/// delegates to it) report about Lemonade's install record, or `None` when the +/// removed runtime had nothing to do with it. +/// +/// Built from the outcome the record handling returned, so each claim is the one +/// that happened: "no longer recorded" only follows a record that was deleted, +/// and a record that could not be deleted or read says so and how to fix it. +pub(crate) fn lemonade_record_report_line(outcome: &RecordCleanup) -> Option { + match outcome { + RecordCleanup::NotAffected => None, + RecordCleanup::Cleared(lemonade) => Some(format!( + "its install at {} was inside this runtime and was removed with it; \ + Lemonade is no longer recorded as installed", + lemonade.display() + )), + RecordCleanup::Failed { + runtime_dir, + record, + error, + } => Some(format!( + "its install at {} was inside this runtime and was removed with it, but \ + Lemonade's install record could not be deleted ({error}); delete {} \ + so Lemonade is not left pointing at files that are gone", + runtime_dir.display(), + record.display() + )), + RecordCleanup::Unreadable(error) => Some(format!( + "Lemonade's install record could not be read ({error}), so it was not \ + checked against this runtime; `rocm engines install lemonade` rewrites it \ + and keeps a current Lemonade install" + )), + } +} + pub(crate) fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { let install_folder = if plan.will_remove_install_root() { format!( @@ -104,6 +154,17 @@ pub(crate) fn print_runtime_uninstall_plan(plan: &RuntimeUninstallPlan) { if plan.clears_default_runtime { report = report.detail("default_runtime", "would be cleared"); } + if plan.will_remove_install_root() + && let Some(lemonade) = plan.lemonade_install_inside.as_ref() + { + report = report.detail( + "lemonade", + format!( + "its install at {} is inside this runtime folder and would be removed with it", + lemonade.display() + ), + ); + } print!("{}", report.render()); } @@ -127,12 +188,23 @@ pub(crate) fn plan_runtime_uninstall( && other.runtime_id.eq_ignore_ascii_case(&manifest.runtime_id) })); let install_root_decision = should_remove_runtime_install_root(&manifest)?; + let holds_lemonade_tree = + rocm_engine_lemonade::install_tree_dirs(&rocm_engine_lemonade::install_tree_root( + paths, + Some(&manifest.install_root.join("engines")), + )) + .iter() + .any(|part| part.exists()); + let lemonade_install_inside = rocm_engine_lemonade::recorded_runtime_dir(paths) + .filter(|dir| rocm_core::runtime_path_is_same_or_inside(dir, &manifest.install_root)); Ok(RuntimeUninstallPlan { manifest, registry_path, was_active, clears_default_runtime, install_root_decision, + lemonade_install_inside, + holds_lemonade_tree, }) } @@ -215,6 +287,8 @@ pub(crate) fn apply_runtime_uninstall( was_active, clears_default_runtime, install_root_decision, + lemonade_install_inside: _, + holds_lemonade_tree, } = plan; let mut removed_install_root = None; @@ -288,11 +362,27 @@ pub(crate) fn apply_runtime_uninstall( } } + // Decided from the folder actually removed, not from the plan: the record + // is only stale once its files are gone. + let lemonade_record = removed_install_root + .as_deref() + .map_or(RecordCleanup::NotAffected, |removed| { + rocm_engine_lemonade::forget_install_record_inside(paths, removed) + }); + // An unreadable record says nothing about this runtime unless the runtime + // could have held Lemonade; otherwise the line would repeat on every runtime + // uninstall for as long as the record stays unreadable. + let lemonade_record = match lemonade_record { + RecordCleanup::Unreadable(_) if !holds_lemonade_tree => RecordCleanup::NotAffected, + other => other, + }; + Ok(RuntimeUninstallResult { runtime_id: manifest.runtime_id, runtime_key: manifest.runtime_key, registry_path, removed_install_root, + lemonade_record, read_only: manifest.read_only, manifest_mismatch: matches!(install_root_decision, InstallRootDecision::ManifestMismatch), was_active, @@ -357,6 +447,180 @@ mod tests { use crate::test_support::write_test_pip_runtime; use crate::tests::test_paths; + /// While a runtime is active, `rocm serve` installs Lemonade inside that + /// runtime's folder and records it there. Uninstalling the runtime deletes + /// the folder; the record must go with it, or Lemonade keeps pointing at + /// files that no longer exist. A record pointing elsewhere is left alone. + #[test] + fn runtime_uninstall_clears_a_lemonade_record_pointing_into_the_removed_folder() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-lemonade"); + let held = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + let other = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-13-0", + "therock-release:gfx1151", + "7.13.0", + 10, + )?; + let tree = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&held.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&tree, 16)?; + crate::test_support::record_lemonade_install(&paths, &tree)?; + let mut config = RocmCliConfig::default(); + + // Removing a runtime that does not hold the tree keeps the record. + let removed = uninstall_runtime(&paths, &mut config, &other.runtime_key)?; + assert_eq!(removed.lemonade_record, RecordCleanup::NotAffected); + assert_eq!( + rocm_engine_lemonade::recorded_runtime_dir(&paths), + Some(tree.join("runtime")) + ); + + let plan = plan_runtime_uninstall(&paths, &config, &held.runtime_key)?; + assert_eq!(plan.lemonade_install_inside, Some(tree.join("runtime"))); + let removed = apply_runtime_uninstall(&paths, &mut config, plan)?; + + assert_eq!( + removed.lemonade_record, + RecordCleanup::Cleared(tree.join("runtime")) + ); + assert!(!tree.exists()); + assert_eq!(rocm_engine_lemonade::recorded_runtime_dir(&paths), None); + let line = lemonade_record_report_line(&removed.lemonade_record).unwrap(); + assert!( + line.contains(&tree.join("runtime").display().to_string()) + && line.ends_with("Lemonade is no longer recorded as installed"), + "{line}" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A record that cannot be read may well name the runtime being removed, so + /// the uninstall cannot tell whether it is left dangling. When the runtime + /// held a Lemonade tree it must say so rather than treat the record as + /// absent, name the cheap command that rewrites it (a plain + /// `rocm engines install lemonade`, shown to do so by + /// `a_plain_install_replaces_a_corrupt_record_and_keeps_the_current_tree` in + /// the engine), not the wiping `--reinstall`, and leave the file alone. + #[test] + fn runtime_uninstall_reports_a_lemonade_record_it_could_not_read() -> Result<()> { + let (root, paths) = test_paths("runtime-uninstall-lemonade-unreadable"); + let held = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + crate::test_support::plant_lemonade_tree( + &rocm_engine_lemonade::install_tree_root( + &paths, + Some(&held.install_root.join("engines")), + ), + 16, + )?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write(&record, b"{ not json")?; + let mut config = RocmCliConfig::default(); + + let removed = uninstall_runtime(&paths, &mut config, &held.runtime_key)?; + + assert!( + matches!(removed.lemonade_record, RecordCleanup::Unreadable(_)), + "{:?}", + removed.lemonade_record + ); + let line = lemonade_record_report_line(&removed.lemonade_record).unwrap(); + assert!(line.contains("could not be read"), "{line}"); + assert!( + line.contains("`rocm engines install lemonade` rewrites it"), + "{line}" + ); + assert!(!line.contains("--reinstall"), "{line}"); + assert!(record.exists()); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The same unreadable record must not be reported on a runtime that never + /// held Lemonade: nothing about that uninstall depends on it, and the line + /// would otherwise repeat on every runtime uninstall. + #[test] + fn an_unreadable_lemonade_record_is_not_reported_for_a_runtime_without_lemonade() -> Result<()> + { + let (root, paths) = test_paths("runtime-uninstall-lemonade-unrelated"); + let runtime = write_test_pip_runtime( + &paths, + "release-pip-gfx1151-7-14-1", + "therock-release:gfx1151", + "7.14.1", + 20, + )?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write(&record, b"{ not json")?; + let mut config = RocmCliConfig::default(); + + let plan = plan_runtime_uninstall(&paths, &config, &runtime.runtime_key)?; + assert!(!plan.holds_lemonade_tree); + let removed = apply_runtime_uninstall(&paths, &mut config, plan)?; + + assert_eq!(removed.lemonade_record, RecordCleanup::NotAffected); + assert_eq!(lemonade_record_report_line(&removed.lemonade_record), None); + assert!( + removed.removed_install_root.is_some(), + "the runtime was removed" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Each outcome's line says only what happened. A failed deletion must + /// name the record to delete and must not claim Lemonade is no longer + /// recorded; no outcome for an unrelated runtime prints anything. (That + /// deleting the named record really clears the condition is shown by + /// `a_record_that_cannot_be_deleted_is_reported_not_swallowed` in the + /// engine.) + #[test] + fn the_lemonade_record_line_matches_what_happened() { + assert_eq!( + lemonade_record_report_line(&RecordCleanup::NotAffected), + None + ); + let cleared = + lemonade_record_report_line(&RecordCleanup::Cleared(PathBuf::from("/rt/lemonade"))) + .unwrap(); + assert!( + cleared.contains("no longer recorded as installed"), + "{cleared}" + ); + let failed = lemonade_record_report_line(&RecordCleanup::Failed { + runtime_dir: PathBuf::from("/rt/lemonade"), + record: PathBuf::from("/data/engines/lemonade/manifests/runtime.json"), + error: "sharing violation".to_owned(), + }) + .unwrap(); + assert!(failed.contains("sharing violation"), "{failed}"); + assert!( + failed.contains("delete /data/engines/lemonade/manifests/runtime.json"), + "{failed}" + ); + assert!(!failed.contains("no longer recorded"), "{failed}"); + } + #[test] fn runtime_uninstall_unregisters_read_only_runtime_without_deleting_external_root() -> Result<()> { diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 731c1afee1..0a43821f68 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -55,7 +55,8 @@ pub(crate) use crate::runtime_registry::{ validate_runtime_manifest_for_activation, }; pub(crate) use crate::runtime_uninstall::{ - InstallRootDecision, should_remove_runtime_install_root, uninstall_runtime, + InstallRootDecision, lemonade_record_report_line, should_remove_runtime_install_root, + uninstall_runtime, }; use crate::runtime_uninstall::{ RuntimeUninstallConfirmation, apply_runtime_uninstall, @@ -257,6 +258,9 @@ pub(crate) fn runtimes(command: Option) -> Result<()> { report = report.detail("folder_removed", "no"); } } + if let Some(line) = lemonade_record_report_line(&result.lemonade_record) { + report = report.detail("lemonade", line); + } if result.default_runtime_cleared { report = report .detail("default_runtime", "cleared") diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 7147245f2c..e78239f08b 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -866,8 +866,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // test reaches Lemonade's backend boundary without real GPU hardware, and the // E2E-only OOM-launch fault injection likewise stands in for the GPU it does // not have. + // + // The scripted free-space override opens the same waiver: it exists to refuse a + // Lemonade install on a drive that cannot hold it, which `rocm serve` only + // reaches past this bail. let scripted_backend_failure = cfg!(feature = "e2e-test-hooks") - && std::env::var_os("ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE").is_some(); + && [ + "ROCM_E2E_LEMONADE_BACKEND_INSTALL_FAILURE", + "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", + ] + .iter() + .any(|variable| std::env::var_os(variable).is_some()); if !cpu_only && !scripted_backend_failure && !e2e_simulate_oom_launch() diff --git a/apps/rocm/src/storage.rs b/apps/rocm/src/storage.rs index 3983cca0ba..b930874699 100644 --- a/apps/rocm/src/storage.rs +++ b/apps/rocm/src/storage.rs @@ -370,10 +370,94 @@ pub(crate) struct RuntimeUsage { pub hold_reason: Option, } +/// Whether Lemonade's install record points at a tree. +/// +/// `Unused` means unused right now: the record points at another tree that +/// really holds an install. It says nothing about the future. Lemonade installs +/// into the active ROCm install's folder, so a copy inside a registered ROCm +/// install is reused the moment that install becomes active, and only the data +/// folder's copy, which no ROCm install owns, is ever offered as reclaimable. A +/// record that names a folder with nothing in it, or that cannot be read, leaves +/// it unknown which copy works, so no tree is called unused. In every case the +/// advice must not tell a user to delete a copy that a later serve would reuse, +/// which would force a reinstall of roughly 25 GiB. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EngineInstallStatus { + /// The install record points here, and the install is there. + InUse, + /// The record points at another tree that holds an install, so nothing uses + /// this one. + Unused, + /// The record names a folder that holds no install, so no copy is in use. + RecordWithoutInstall, + /// A record exists but could not be read, so it is not known which copy is + /// in use. + RecordUnreadable, + /// Lemonade has no install record at all. + NotRecorded, +} + +/// Decide a tree's status from the install record. +fn classify_engine_install( + record: &rocm_engine_lemonade::InstallRecord, + tree: &Path, +) -> EngineInstallStatus { + use rocm_engine_lemonade::InstallRecord; + match record { + InstallRecord::NotRecorded => EngineInstallStatus::NotRecorded, + InstallRecord::Unreadable(_) => EngineInstallStatus::RecordUnreadable, + InstallRecord::Recorded { + holds_install: false, + .. + } => EngineInstallStatus::RecordWithoutInstall, + InstallRecord::Recorded { runtime_dir, .. } + if rocm_core::runtime_path_is_same_or_inside(runtime_dir, tree) => + { + EngineInstallStatus::InUse + } + InstallRecord::Recorded { .. } => EngineInstallStatus::Unused, + } +} + +/// One Lemonade install tree found on disk. +/// +/// A self-managed engine installs into the active ROCm runtime's own folder +/// when one is active, and into the data folder otherwise, so the same host +/// can carry a full copy in each. Before this row existed the only trace of a +/// second copy was a larger runtime size and data-folder total. +#[derive(Debug, Clone, Serialize)] +pub(crate) struct EngineInstallUsage { + pub engine: String, + pub path: PathBuf, + /// The runtime whose folder holds this tree, `None` for the data folder. + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_key: Option, + pub size_bytes: u64, + pub size_complete: bool, + pub status: EngineInstallStatus, + /// Whether the report offers this copy for deletion: unused right now and + /// not inside a registered ROCm install, where it would be reused when that + /// install becomes active. + pub reclaimable: bool, + /// Why the status could not be decided, for the two record-problem states. + #[serde(skip_serializing_if = "Option::is_none")] + pub record_note: Option, + /// Whether `rocm runtimes uninstall` deletes this tree along with its + /// runtime. Decided by the same rule uninstall applies + /// (`should_remove_runtime_install_root`): an external or imported runtime + /// is only unregistered, so its Lemonade copy stays. `false` for the data + /// folder's tree, which belongs to no runtime. + pub removed_with_runtime: bool, +} + #[derive(Debug, Clone, Serialize)] pub(crate) struct StorageReport { pub runtimes: Vec, pub runtimes_total_bytes: u64, + /// Engine installs ROCm CLI made. A tree inside a runtime folder is also + /// part of that runtime's size above. + pub engines: Vec, pub rocm_cli: Vec, /// Caches ROCm CLI uses but does not own. Reported, never deleted. pub shared_with_other_tools: Vec, @@ -419,6 +503,81 @@ fn shared_model_cache_dir() -> Option { Some(home.join(".cache").join("huggingface")) } +/// Every Lemonade tree ROCm CLI could have installed: the data folder's, and +/// one per registered runtime (where it goes while that runtime is active). +/// Only the install parts are measured (`install_tree_dirs`), not the record, +/// logs and server state that share the data folder's tree. +fn lemonade_install_usages( + paths: &AppPaths, + manifests: &[therock::InstalledRuntimeManifest], +) -> Vec { + let record = rocm_engine_lemonade::install_record(paths); + let record_note = match &record { + rocm_engine_lemonade::InstallRecord::Unreadable(error) => Some(format!( + "Lemonade's install record could not be read ({error}), so it is not known which copy is in use" + )), + rocm_engine_lemonade::InstallRecord::Recorded { + runtime_dir, + holds_install: false, + } => Some(format!( + "Lemonade's install record names {}, which holds no install, so no copy is in use", + runtime_dir.display() + )), + _ => None, + }; + let candidates = std::iter::once(( + None, + rocm_engine_lemonade::install_tree_root(paths, None), + false, + )) + .chain(manifests.iter().map(|manifest| { + ( + Some(manifest.runtime_key.clone()), + rocm_engine_lemonade::install_tree_root( + paths, + Some(&manifest.install_root.join("engines")), + ), + should_remove_runtime_install_root(manifest) + .is_ok_and(InstallRootDecision::should_remove), + ) + })); + candidates + .filter_map(|(runtime_key, path, removed_with_runtime)| { + let parts = rocm_engine_lemonade::install_tree_dirs(&path); + let present = parts + .iter() + .filter(|part| part.exists()) + .collect::>(); + if present.is_empty() { + return None; + } + let (size_bytes, size_complete) = + present + .iter() + .fold((0_u64, true), |(bytes, complete), part| { + let measured = measure_path(part); + ( + bytes.saturating_add(measured.bytes), + complete && measured.complete, + ) + }); + let status = classify_engine_install(&record, &path); + let reclaimable = status == EngineInstallStatus::Unused && runtime_key.is_none(); + Some(EngineInstallUsage { + engine: "lemonade".to_owned(), + path, + runtime_key, + size_bytes, + size_complete, + status, + reclaimable, + record_note: record_note.clone(), + removed_with_runtime, + }) + }) + .collect() +} + pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result { let manifests = therock::load_runtime_manifests(paths)?; let marker = read_active_runtime_marker(paths); @@ -532,9 +691,12 @@ pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result String { format_bytes(report.runtimes_total_bytes) ); + let _ = writeln!(output); + let _ = writeln!(output, "Engine installs:"); + if report.engines.is_empty() { + let _ = writeln!(output, " none"); + } + for usage in &report.engines { + let size = if usage.size_complete { + format_bytes(usage.size_bytes) + } else { + format!( + "{} or more (part unreadable)", + format_bytes(usage.size_bytes) + ) + }; + let location = usage.runtime_key.as_deref().map_or_else( + || "data folder".to_owned(), + |key| format!("inside ROCm install {key}"), + ); + let _ = writeln!( + output, + " - {} ({location}): {size} ({})", + usage.engine, + usage.path.display() + ); + let status = match usage.status { + EngineInstallStatus::InUse => "in use", + EngineInstallStatus::Unused => "not used right now; Lemonade runs from another install", + EngineInstallStatus::RecordWithoutInstall => { + "not known to be used; the install record names a folder with no install" + } + EngineInstallStatus::RecordUnreadable => { + "not known to be used; the install record could not be read" + } + EngineInstallStatus::NotRecorded => "not recorded as installed", + }; + let _ = writeln!(output, " status: {status}"); + if usage.reclaimable { + let _ = writeln!( + output, + " note: reclaimable; Lemonade installs into the active ROCm install's folder, so this copy is only used while no ROCm install is active. Nothing removes it automatically: delete its runtime, downloads and extract folders to free the space" + ); + } else if usage.status == EngineInstallStatus::Unused { + let _ = writeln!( + output, + " note: reused if this ROCm install becomes the active one; not marked reclaimable" + ); + } + if let Some(note) = usage.record_note.as_deref() { + let _ = writeln!(output, " note: {note}; not marked reclaimable"); + } + if usage.runtime_key.is_some() { + let note = if usage.removed_with_runtime { + "also counted in that ROCm install's size above, and removed with it" + } else { + "also counted in that ROCm install's size above; ROCm CLI does not delete that install's folder (external or imported), so uninstalling it leaves this copy" + }; + let _ = writeln!(output, " note: {note}"); + } + } + let _ = writeln!(output); let _ = writeln!(output, "ROCm CLI folders:"); for usage in &report.rocm_cli { @@ -959,6 +1181,11 @@ pub(crate) fn storage(command: Option) -> Result<()> { format_bytes(entry.size_bytes), path.display() ); + if let Some(line) = + crate::runtimes_cmd::lemonade_record_report_line(&result.lemonade_record) + { + println!(" lemonade: {line}"); + } } else { println!( "removed the registry entry for {}, but left its folder {} in place", @@ -1468,6 +1695,299 @@ mod tests { Ok(()) } + /// The release-test layout (ROCMAI-825): Lemonade installed into the data + /// folder, then a runtime activated and `rocm serve` installing a second full + /// copy inside it, with the record now pointing at the second. Both trees + /// must be listed with their sizes, and the data folder's flagged as unused. + #[test] + fn report_lists_both_lemonade_installs_and_flags_the_unused_one() -> Result<()> { + let (root, paths) = test_paths("report-lemonade"); + let runtime = install_fixture(&paths, manifest("in-use", "gfx1151", "7.14.1", 20), 4096)?; + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + let per_runtime = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + crate::test_support::plant_lemonade_tree(&per_runtime, 5000)?; + crate::test_support::record_lemonade_install(&paths, &per_runtime)?; + // Record, logs and server state share the data folder's tree; they are + // not part of the install and must not be counted as reclaimable. + std::fs::create_dir_all(base.join("state"))?; + std::fs::write(base.join("state").join("big.log"), vec![0_u8; 100_000])?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!(report.engines.len(), 2, "{:?}", report.engines); + let base_row = &report.engines[0]; + assert_eq!(base_row.path, base); + assert_eq!(base_row.runtime_key, None); + assert_eq!(base_row.status, EngineInstallStatus::Unused); + assert!(base_row.size_bytes >= 3000 && base_row.size_bytes < 100_000); + let runtime_row = &report.engines[1]; + assert_eq!(runtime_row.path, per_runtime); + assert_eq!(runtime_row.runtime_key.as_deref(), Some("in-use")); + assert_eq!(runtime_row.status, EngineInstallStatus::InUse); + assert!(runtime_row.size_bytes >= 5000); + + let rendered = render_report(&report); + assert!( + rendered.contains("- lemonade (data folder): "), + "{rendered}" + ); + assert!( + rendered.contains("status: not used right now; Lemonade runs from another install"), + "{rendered}" + ); + assert!( + rendered.contains(&format!( + "- lemonade (inside ROCm install in-use): {} ({})", + format_bytes(runtime_row.size_bytes), + per_runtime.display() + )), + "{rendered}" + ); + assert!(rendered.contains("note: reclaimable"), "{rendered}"); + assert!(base_row.reclaimable && !runtime_row.reclaimable); + assert!( + rendered + .contains("also counted in that ROCm install's size above, and removed with it"), + "a runtime ROCm CLI owns is deleted with its copy:\n{rendered}" + ); + + // With no record at all, nothing is called unused: the next install may + // well reuse a current tree. + std::fs::remove_file(paths.engine_manifests_dir("lemonade").join("runtime.json"))?; + let report = build_report(&paths, &RocmCliConfig::default())?; + assert!( + report + .engines + .iter() + .all(|row| row.status == EngineInstallStatus::NotRecorded) + ); + assert!(!render_report(&report).contains("note: reclaimable")); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// Two registered runtimes that each hold a Lemonade copy, with the record + /// pointing at one. The other copy is unused right now but is reused as soon + /// as its runtime becomes active, so it must not be offered for deletion: + /// following that advice would force a ~25 GiB reinstall. Only the data + /// folder's copy, which no runtime owns, is reclaimable. + #[test] + fn a_copy_inside_another_registered_runtime_is_not_offered_for_deletion() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-two-runtimes"); + let active = install_fixture(&paths, manifest("active", "gfx1151", "7.14.1", 20), 4096)?; + let idle = install_fixture(&paths, manifest("idle", "gfx1151", "7.13.0", 10), 4096)?; + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + let tree_of = |runtime: &therock::InstalledRuntimeManifest| { + rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ) + }; + crate::test_support::plant_lemonade_tree(&base, 3000)?; + crate::test_support::plant_lemonade_tree(&tree_of(&active), 5000)?; + crate::test_support::plant_lemonade_tree(&tree_of(&idle), 6000)?; + crate::test_support::record_lemonade_install(&paths, &tree_of(&active))?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + let row = |runtime_key: Option<&str>| { + report + .engines + .iter() + .find(|row| row.runtime_key.as_deref() == runtime_key) + .expect("the copy is listed") + }; + assert_eq!(row(Some("active")).status, EngineInstallStatus::InUse); + assert_eq!(row(Some("idle")).status, EngineInstallStatus::Unused); + assert!( + !row(Some("idle")).reclaimable, + "reused when `idle` is activated" + ); + assert!( + row(None).reclaimable, + "no runtime owns the data folder's copy" + ); + let rendered = render_report(&report); + assert_eq!( + rendered.matches("note: reclaimable").count(), + 1, + "only the data folder's copy is offered:\n{rendered}" + ); + assert!( + rendered.contains( + "note: reused if this ROCm install becomes the active one; not marked reclaimable" + ), + "{rendered}" + ); + // Reporting only: every copy is still on disk. + assert!(tree_of(&idle).join("runtime").exists() && base.join("runtime").exists()); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// A record that names a runtime folder holding no install (a runtime + /// uninstalled by a build that left the record behind) means nothing is in + /// use. The surviving data-folder copy is then the only one a later serve can + /// reuse, so the report must not call it unused or reclaimable: following that + /// advice would force a reinstall of roughly 25 GiB. + #[test] + fn a_record_naming_a_folder_with_no_install_does_not_make_the_survivor_reclaimable() + -> Result<()> { + let (root, paths) = test_paths("report-lemonade-dangling"); + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + let gone = paths + .data_dir + .join("runtimes") + .join("wheel") + .join("removed"); + crate::test_support::record_lemonade_install( + &paths, + &gone.join("engines").join("lemonade"), + )?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!(report.engines.len(), 1, "{:?}", report.engines); + let row = &report.engines[0]; + assert_eq!(row.status, EngineInstallStatus::RecordWithoutInstall); + let rendered = render_report(&report); + assert!(!rendered.contains("note: reclaimable"), "{rendered}"); + assert!(!rendered.contains("status: not used"), "{rendered}"); + assert!( + rendered.contains("install record names a folder with no install"), + "{rendered}" + ); + assert!( + rendered.contains("holds no install, so no copy is in use; not marked reclaimable") + ); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// An unparseable record says nothing about which copy works, so it is not + /// "no record" and no copy is called unused. + #[test] + fn an_unreadable_record_leaves_every_copy_unmarked() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-unreadable"); + let base = rocm_engine_lemonade::install_tree_root(&paths, None); + crate::test_support::plant_lemonade_tree(&base, 3000)?; + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + std::fs::create_dir_all(record.parent().expect("record has a parent"))?; + std::fs::write(&record, b"{ not json")?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + assert_eq!( + report.engines[0].status, + EngineInstallStatus::RecordUnreadable + ); + let rendered = render_report(&report); + assert!(!rendered.contains("note: reclaimable"), "{rendered}"); + assert!(rendered.contains("could not be read"), "{rendered}"); + assert!( + !rendered.contains("not recorded as installed"), + "{rendered}" + ); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + + /// The status rule on its own, over every record state and tree position. + #[test] + fn engine_install_status_follows_the_record() { + use rocm_engine_lemonade::InstallRecord; + let tree = PathBuf::from("data").join("engines").join("lemonade"); + let inside = InstallRecord::Recorded { + runtime_dir: tree.join("runtime"), + holds_install: true, + }; + let elsewhere = InstallRecord::Recorded { + runtime_dir: PathBuf::from("elsewhere").join("runtime"), + holds_install: true, + }; + let empty = InstallRecord::Recorded { + runtime_dir: PathBuf::from("elsewhere").join("runtime"), + holds_install: false, + }; + assert_eq!( + classify_engine_install(&inside, &tree), + EngineInstallStatus::InUse + ); + assert_eq!( + classify_engine_install(&elsewhere, &tree), + EngineInstallStatus::Unused + ); + assert_eq!( + classify_engine_install(&empty, &tree), + EngineInstallStatus::RecordWithoutInstall + ); + // A record at this tree whose server binary is gone is not "in use". + let hollow = InstallRecord::Recorded { + runtime_dir: tree.join("runtime"), + holds_install: false, + }; + assert_eq!( + classify_engine_install(&hollow, &tree), + EngineInstallStatus::RecordWithoutInstall + ); + assert_eq!( + classify_engine_install(&InstallRecord::Unreadable("x".to_owned()), &tree), + EngineInstallStatus::RecordUnreadable + ); + assert_eq!( + classify_engine_install(&InstallRecord::NotRecorded, &tree), + EngineInstallStatus::NotRecorded + ); + } + + /// Uninstalling an external or imported runtime only unregisters it, so its + /// Lemonade copy stays. The row must not say it is "removed with" the + /// runtime; the claim comes from the rule uninstall itself applies. + #[test] + fn a_read_only_runtimes_lemonade_copy_is_not_said_to_be_removed_with_it() -> Result<()> { + let (root, paths) = test_paths("report-lemonade-read-only"); + let mut record = manifest("external", "gfx1151", "7.14.1", 20); + record.read_only = true; + let runtime = install_fixture(&paths, record, 4096)?; + let tree = rocm_engine_lemonade::install_tree_root( + &paths, + Some(&runtime.install_root.join("engines")), + ); + crate::test_support::plant_lemonade_tree(&tree, 5000)?; + crate::test_support::record_lemonade_install(&paths, &tree)?; + + let report = build_report(&paths, &RocmCliConfig::default())?; + + let row = report + .engines + .iter() + .find(|row| row.runtime_key.as_deref() == Some("external")) + .expect("the runtime's copy is listed"); + assert!(!row.removed_with_runtime); + let rendered = render_report(&report); + assert!(!rendered.contains("and removed with it"), "{rendered}"); + assert!( + rendered.contains("does not delete that install's folder (external or imported)"), + "{rendered}" + ); + // The claim matches what uninstall then does to that copy. + let mut config = RocmCliConfig::default(); + crate::runtimes_cmd::uninstall_runtime(&paths, &mut config, "external")?; + assert!(tree.join("runtime").exists(), "uninstall deleted the copy"); + + let _ = std::fs::remove_dir_all(root); + Ok(()) + } + /// `rocm serve --managed` leaves one JSON record per launch, kept after the /// server exits, and no `rocm storage` command removes them. The report /// never named the folder, so a user asking what is on disk could not see diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 1987de3679..caaa736810 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -485,3 +485,43 @@ mod tests { assert_eq!(raised_case_count(Some("lots"), 256), 256); } } + +/// Plant an extracted Lemonade tree under `tree_root` the way an install leaves +/// one: the server binary and `payload_bytes` of backend files. Black-box on +/// purpose (plain files, not the engine's own writer), so a test reads it the +/// way the CLI reads a real install. +pub(crate) fn plant_lemonade_tree(tree_root: &std::path::Path, payload_bytes: usize) -> Result<()> { + let runtime_dir = tree_root.join("runtime"); + let backend = runtime_dir.join("bin").join("llamacpp").join("rocm-stable"); + fs::create_dir_all(&backend)?; + let lemond = if rocm_core::runtime_is_windows() { + "lemond.exe" + } else { + "lemond" + }; + fs::write(runtime_dir.join(lemond), b"lemond")?; + fs::write(backend.join("payload.bin"), vec![0_u8; payload_bytes])?; + Ok(()) +} + +/// Write Lemonade's install record pointing at `tree_root`'s runtime, in the +/// schema the engine writes (`/engines/lemonade/manifests/runtime.json`). +pub(crate) fn record_lemonade_install(paths: &AppPaths, tree_root: &std::path::Path) -> Result<()> { + let runtime_dir = tree_root.join("runtime"); + let record = paths.engine_manifests_dir("lemonade").join("runtime.json"); + fs::create_dir_all(record.parent().expect("record has a parent"))?; + fs::write( + &record, + serde_json::to_vec_pretty(&serde_json::json!({ + "env_id": "lemonade-embeddable-test", + "version": "test", + "runtime_dir": runtime_dir, + "lemond": runtime_dir.join("lemond"), + "lemonade": runtime_dir.join("lemonade"), + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + }))?, + )?; + Ok(()) +} diff --git a/docs/architecture.md b/docs/architecture.md index 0a0ba71bf7..afff7b3ac0 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -85,8 +85,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_install_finalize/torch_alignment.rs`](../apps/rocm/src/runtime_install_finalize/torch_alignment.rs) — full domain extraction, a child of `runtime_install_finalize.rs` that depends on its parent's device check (`RuntimeDeviceCheck`, `device_check_verdict`) and on `runtime_manifest_for_selector` in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): what to do about the torch an engine install finds in a managed runtime. `plan_torch_alignment` decides whether the SDK's torch must be realigned to the engine's pin, `align_runtime_torch` runs the realignment, `classify_retained_torch` decides when a torch that already works is kept as an intended divergence instead, and the `render_*` and `report_*` functions with `TorchAlignment` and `TorchRetention` print and log the outcome. The manifest readers (`sdk_torch_build_for_key`, `sdk_torch_build_from_manifest`, `runtime_python_for_key`, `runtime_library_paths_for_key`, `runtime_index_url_for_key`) live here too. The parent stays the caller (`settle_engine_install` and `maybe_auto_install_sdk_preferred_engine`), so it could not be a crate-level sibling without a cycle with the parent's device check. The items the parent calls are `pub(super)`: `TorchAlignment`, `TorchRetention`, `report_torch_alignment`, `report_torch_retention`, `classify_retained_torch`, `retained_diverged_package`, `deliberately_diverged_package`, `sdk_torch_build_for_key`, `runtime_python_for_key` and `runtime_library_paths_for_key`; the rest is private. Its tests cover planning, rendering, the opt-out and the retention rules, and reach the parent's private engine-dependency check as a descendant of it. - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing them (`write_runtime_manifest`, the one routine every way of registering a runtime goes through: an SDK install, `rocm runtimes import`, `rocm runtimes adopt` and the recovery of the runtime `rocm setup` created. It writes the registry entry, and for a runtime ROCm CLI owns (`InstalledRuntimeManifest::owns_install_root`, the same rule uninstall and prune use to decide whether they may delete the folder) the `.rocm-cli-runtime.json` in its install root; an imported or adopted runtime is someone else's folder, so it gets the registry entry only. The registry entry is written last, so a failed write of either file never leaves a new entry behind and an error that says a runtime could not be registered or restored is true. Its one parameter is `ManifestOverwrite`: an install replaces its own entry, import and adopt refuse an existing one unless `--replace` is given, and recovery refuses so it only fills a missing entry. Neither file is written atomically) and loading them (`load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it; the tarball one asks `rocm_core::install_root_has_payload` whether the root holds a payload). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). It also holds `paths_equivalent` and `normalize_path_for_compare`, the path comparison that [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) and `runtimes_cmd.rs` need, and the on-disk active-runtime marker: `ActiveRuntimeMarker` (user state, so its serde shape is as fixed as the manifest's; its fields are `pub(crate)` because `runtimes_cmd.rs` builds it on activation and rollback), `active_runtime_marker_path` and `active_runtime_marker_matches`, which uninstall uses to decide whether the runtime it removes was the active one. `write_active_runtime_marker` stays in `runtimes_cmd.rs` with the activation transaction. `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`cli.rs`](../apps/rocm/src/cli.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. -- [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. -- [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` does). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root` and `InstallRootDecision` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. +- [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs) — full domain extraction, crate-level (split out of [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)): `rocm runtimes uninstall` and the storage pruning that removes a registered runtime. `plan_runtime_uninstall` builds the plan (`RuntimeUninstallPlan`, with the `InstallRootDecision` for its folder), `print_runtime_uninstall_plan` shows it, `confirm_and_revalidate_runtime_uninstall` asks and then checks that nothing changed while the prompt was open (`revalidate_runtime_uninstall_plan`), and `apply_runtime_uninstall` unregisters the runtime, clears the active and default state that named it and removes its install root when `should_remove_runtime_install_root` says ROCm CLI owns it (`local_runtime_manifest_matches`, `ensure_runtime_install_root_is_safe_to_remove`). `RuntimeUninstallPlan` also records whether Lemonade's install record points inside the runtime's folder (`lemonade_install_inside`, from `rocm_engine_lemonade::recorded_runtime_dir`), which `print_runtime_uninstall_plan` reports, and whether the folder holds a Lemonade tree at all (`holds_lemonade_tree`), which decides whether an unreadable record is worth reporting; `apply_runtime_uninstall` clears that record once the folder is removed (`RuntimeUninstallResult::lemonade_record`, a `RecordCleanup` that also carries a failed deletion or an unreadable record, worded by `lemonade_record_report_line`). `uninstall_runtime` is the non-interactive entry point. It reads the registry, the active-runtime marker and the path comparison from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and does not depend on `runtimes_cmd.rs`, which calls into it from `runtimes()` and re-exports `uninstall_runtime`, `should_remove_runtime_install_root`, `InstallRootDecision` and `lemonade_record_report_line`. The planning and apply entry points, `RuntimeUninstallPlan`, `RuntimeUninstallConfirmation` and `RuntimeUninstallResult` (with its fields) are `pub(crate)` for `runtimes()`; `revalidate_runtime_uninstall_plan` and the folder-safety helpers are private. Its tests cover planning, revalidation, the folder-removal rules and uninstalling a read-only imported runtime; the uninstall tests that first activate a runtime stay with the activation tests in `runtimes_cmd.rs`. +- [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback handlers (the activation transaction, including `write_active_runtime_marker`), plus the runtime list rendering; the import and adopt handlers it dispatches to live in [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs) and the uninstall plan, confirmation and apply in [`runtime_uninstall.rs`](../apps/rocm/src/runtime_uninstall.rs); registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` does). `RuntimeActivationResult` is `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary`; `uninstall_runtime`, `should_remove_runtime_install_root`, `InstallRootDecision` and `lemonade_record_report_line` (from `runtime_uninstall.rs`) and `ActiveRuntimeMarker` and `active_runtime_marker_path` (from `runtime_registry.rs`) are re-exported for [`storage.rs`](../apps/rocm/src/storage.rs) and [`config_cmd.rs`](../apps/rocm/src/config_cmd.rs); `recover_setup_runtime_registration`, `newest_installed_runtime_manifest` and `setup_runtime_paths` (from `runtime_adopt.rs`) and `normalize_path_for_compare` (from `runtime_registry.rs`) are re-exported for the modules that import them from here, such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs), [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) and [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs). [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) is deliberately not one of them: it imports from `runtime_adopt.rs` and `runtime_registry.rs` directly, because going through this module would lead back to it via `runtime_services.rs`. The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. @@ -99,7 +99,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. -- [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. +- [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. Its report also lists each Lemonade install tree (`EngineInstallUsage`, built by `lemonade_install_usages` from the read-only view in [`footprint.rs`](../engines/lemonade/src/footprint.rs)) with its size and whether the install record uses it (`EngineInstallStatus`, decided by `classify_engine_install`), and whether `rocm runtimes uninstall` would delete it with its runtime. - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and reads the os-release every one of these plans is built from through `rocm_core::hardware_root::real_os_release`, which has no copy here. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `rocm_core::hardware_root::real_os_release` and the three OpenMPI/libatomic/libnuma install hints ignore a simulated host root, and a scan that no source file of the binary reads os-release by hand. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. @@ -427,6 +427,7 @@ These modules are private `mod` declarations — [`lib.rs`](../crates/e2e-report - [`backend_alignment.rs`](../engines/lemonade/src/backend_alignment.rs) — ROCm-SDK-to-llama.cpp-backend Tier 1/Tier 2/revert alignment, including the `backend_versions.json` pin handling and the set-aside/restore of ROCm backend directories around a forced reinstall. It uses `llamacpp_backend.rs`; that module does not import it back. - [`device_policy.rs`](../engines/lemonade/src/device_policy.rs) — device-policy and GPU-selection handling: the GPU-required policy check (a CPU-only request is refused, never downgraded), `--device`/`--gpu` argument parsing, probing and resolving the GPU ordinal(s) to pin before a launch, the device availability reported by `detect`, and the device string reported for a loaded model. A leaf: it imports no other crate module. - [`direct_llama.rs`](../engines/lemonade/src/direct_llama.rs) — Hugging Face checkpoint/GGUF resolution and the direct-llama-server serve path (`serve_direct_llama_server`, which runs a packaged `llama-server` on a GGUF and waits for `/v1/models` to report the model). It imports `process.rs`; `process.rs` no longer imports it. +- [`footprint.rs`](../engines/lemonade/src/footprint.rs) — what an install costs on disk: the pre-install notice and free-space check, plus the read-only view of Lemonade's install trees and record that `apps/rocm` uses for `rocm storage` and `rocm runtimes uninstall` (re-exported at the crate root). - [`install.rs`](../engines/lemonade/src/install.rs) — embeddable-runtime download, extraction orchestration (the cache layer is in `archive_cache.rs`, the extractor in `archive_extract.rs`), and manifest persistence, plus the `LemonadeRuntime` handle and `resolve_runtime`, which wrap the persisted manifest (so `state.rs`, `process.rs` and `direct_llama.rs` reach it without importing `backend_alignment.rs`). - [`llamacpp_backend.rs`](../engines/lemonade/src/llamacpp_backend.rs) — llama.cpp backend discovery and selection: asking `lemonade backends` what the host supports, choosing the best GPU backend (`rocm` before `vulkan`, never `cpu`), installing it with one retry, the `ROCM_LLAMACPP_BACKEND_DIRS` name list, and `find_llama_server_binary_for_backend`. It imports `direct_llama.rs` for `find_binary_in`; `direct_llama.rs` does not import it. With this module the engine's modules form no import cycle outside test code. - [`process.rs`](../engines/lemonade/src/process.rs) — `lemond` spawn, env/path setup, readiness/health polling, and the background relaunch of the `serve-http` command (`serve_http_command_args`, `spawn_serve_http_background`). The direct `llama-server` serve loop is in `direct_llama.rs`. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c8528d7118..106da7b3c9 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -333,6 +333,36 @@ ROCm backend directory under `bin/llamacpp/`, and run - `therock.version` and `llamacpp.rocm-stable` in `backend_versions.json` hold the packaged values again, and the unrelated key is still there. +To check what a Lemonade install says about its size (no download needed for +the refusal): with a debug build, set +`ROCM_E2E_LEMONADE_FREE_SPACE_BYTES=1048576` (1 MiB free), then in a fresh data +folder run `rocm engines install lemonade --yes`. Expected result: + +- Before anything else it prints `Installing Lemonade into ` + and a line giving the ROCm backend's download (up to about 4.4 GiB), disk use + (up to about 25 GiB) and the free space on that drive. +- It then fails with `not enough free disk space to install Lemonade: ... + Nothing was downloaded.`, and `` has no `downloads` or `runtime` + subfolder. +- With `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES=2147483648` (2 GiB) instead, it + warns that only 2.0 GiB is free for a backend of up to about 4.4 GiB and + carries on: Lemonade picks the backend after this point and a smaller one may + fit. (Stop it before the download if you do not want one.) + +With room to spare, on Windows from an interactive console: install Lemonade +with no runtime active, then `rocm install sdk`, then `rocm serve `. +Expected result: + +- The serve prints the install notice for the runtime's own + `engines\lemonade` folder, and says the data folder's copy is not reused. +- Lemonade's own backend download progress (`[1/2] ...`, `Total: ...`) appears + in the console instead of a silent wait. +- `rocm storage report` lists both copies under `Engine installs`, with the data + folder's one `not used right now` (and reclaimable) and the runtime's one `in use`. +- `rocm runtimes uninstall ` says Lemonade's install was inside the + runtime and removed with it, and a following `rocm storage report` shows the + remaining copy as `not recorded as installed`. + While the log stream is attached, verify detach and stop behave differently: - Press `Ctrl+D`. The stream ends and the shell prompt returns, printing a diff --git a/docs/testing.md b/docs/testing.md index cbe31a2650..91a165198b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1395,6 +1395,40 @@ cargo test -p rocmd event_collector cargo test -p rocmd event_dispatcher ``` +## Lemonade Installs On Disk + +`features/lemonade_installs.feature` covers what a user is told about the +Lemonade copies ROCm CLI keeps (ROCMAI-825), all on the mock lane: + +- `lemonade-installs-01` and `-02` reach the engine's pre-install free-space + refusal from `rocm engines install lemonade` and from `rocm serve`, with 1 MiB + reported free (below what any Lemonade install needs). Room for Lemonade but + not for the multi-gigabyte ROCm backend only warns, which the unit tests cover + because a scenario would have to start the download. They use + `ROCM_E2E_LEMONADE_FREE_SPACE_BYTES`, compiled only into the + `rocm/e2e-test-hooks` build `cargo xtask e2e` makes, and check against the + disk that nothing was downloaded. That variable also waives serve's no-GPU + refusal, as serve-18's failure hook does. +- `lemonade-installs-03` plants a data-folder copy and a copy inside a runtime + and checks `rocm storage report` lists both and flags the unused one. +- `lemonade-installs-04` uninstalls that runtime and checks the install record + is cleared. +- `lemonade-installs-05` and `-06` cover the two cases the report must not + overstate: a record naming a folder with no install (the surviving copy is + not offered for deletion) and a read-only runtime (its copy is not said to be + removed with it, and uninstall then leaves it). A copy inside a registered + runtime is never marked reclaimable, because it is reused when that runtime + becomes active; two registered runtimes are covered by a unit test. + +Run them alone with `cargo xtask e2e -- -n "lemonade-installs-"`. + +The Windows console fix for the backend download's progress is unit-tested +through the real command builder (`backend_install_command` in +[process.rs](../engines/lemonade/src/process.rs)), which receives the console +state and the hide step, so a dropped or unconditional hide turns the test red. +Whether the output actually reaches a real console is a manual check in +[manual-testing.md](manual-testing.md). + ## Uninstall And Download Removal `rocm uninstall` and `rocm storage remove-downloads` both end in the same diff --git a/engines/lemonade/src/footprint.rs b/engines/lemonade/src/footprint.rs new file mode 100644 index 0000000000..9acdf26ccb --- /dev/null +++ b/engines/lemonade/src/footprint.rs @@ -0,0 +1,1010 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What a Lemonade install costs on disk, and where its trees live. +//! +//! A fresh install is a multi-gigabyte download and a tree several times that +//! size: on a Windows gfx1151 host, Lemonade's `llamacpp:rocm` backend fetched +//! 4.7 GB (Lemonade's own `Total:` line) and the finished tree measured 24.3 GiB, +//! almost all of it the ROCm runtime Lemonade bundles with the backend. Before +//! this module the only thing a user saw was a one-line "Preparing..." and then, +//! from `rocm serve`, up to ten silent minutes. [`plan_install_preflight`] decides +//! whether an install is about to write a new tree, and [`InstallPreflight`] +//! renders what will be installed, where, how big it is, and refuses up front +//! when the drive cannot hold even the download. +//! +//! The second half is the read-only view `apps/rocm` needs to report Lemonade +//! trees (`rocm storage`) and to forget an install record whose tree a runtime +//! uninstall just deleted. The record and the tree layout belong to this crate, +//! so the knowledge of both stays here rather than being re-parsed by the CLI. + +use std::fs; +use std::path::{Path, PathBuf}; + +use rocm_core::{ + AppPaths, format_bytes, insufficient_space_message, runtime_path_is_same_or_inside, +}; + +use crate::install::{ + installed_runtime_version, lemonade_root, lemond_path_in, manifest_path, needs_extraction, + runtime_dir_in, +}; + +/// Download size of an install that fetches the llama.cpp ROCm backend: the +/// embeddable archive plus the backend. Lemonade printed `Total: 4.7 GB` +/// (decimal) for the backend on a Windows gfx1151 host; the embeddable archive +/// adds ~5 MB. +/// +/// An upper bound, never a promise. The backend is chosen after this preflight +/// (Lemonade reports per-GPU support, and `backend_alignment.rs` falls back to +/// `vulkan` where its ROCm build is unsupported), and the figure varies with the +/// GPU family and Lemonade's pins. So it is printed as "up to about" and only +/// ever warns; see [`MIN_INSTALL_BYTES`] for what can refuse. +pub(crate) const FRESH_INSTALL_DOWNLOAD_BYTES: u64 = 4_700_000_000; + +/// Disk space an install that fetches the ROCm backend ends up using: 24.3 GiB +/// measured for the same install, rounded up. An estimate and an upper bound, so +/// it only ever warns (the `rocm_core::disk_space` convention: exact +/// requirements may refuse, estimates only warn). +pub(crate) const FRESH_INSTALL_DISK_BYTES: u64 = 25 * 1024 * 1024 * 1024; + +/// What every Lemonade install needs whichever backend is chosen: the embeddable +/// archive (4.8 MB) plus the tree it extracts (10.6 MiB outside `bin`, measured on +/// the same host), rounded up. The one figure that is certain, so the one thing +/// a refusal may rest on: a drive without this much room cannot hold Lemonade at +/// all, however small a backend it picks. +pub(crate) const MIN_INSTALL_BYTES: u64 = 16 * 1024 * 1024; + +/// What an install is about to do, decided before anything is written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct InstallPreflight { + /// The folder the install writes into (`/runtime`, its downloads...). + pub(crate) target_root: PathBuf, + /// The Lemonade install the record points at, when it really holds one and + /// is a different folder: the install is not going to reuse it. Never set + /// for a record whose folder no longer holds an install, because nothing is + /// installed there to be "not reused". + pub(crate) other_install: Option, + /// Whether `rocm storage` lists `other_install`, so the notice may point at + /// it. Storage lists only the data folder's tree and trees inside registered + /// ROCm installs; a recorded install anywhere else is announced but not + /// listed, and a pointer to it would be false. + pub(crate) other_install_listed: bool, + /// Bytes the install deletes before it extracts: the existing runtime + /// folder when this install replaces it (`--reinstall`, a new Lemonade + /// version, or a missing server binary), whose space is free by the time + /// the download starts. Zero when the tree is kept. + pub(crate) replaced_bytes: u64, + /// Free bytes on the drive holding `target_root`, when known. + pub(crate) available: Option, +} + +/// Outcome of comparing the preflight's free space with the install's size. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SpaceVerdict { + /// Enough room, or free space could not be determined (never block on that). + Fits, + /// Room for the download but likely not for the finished tree. + Tight(String), + /// Not even room for the download: the install cannot finish. + Refuse(String), +} + +/// Whether installing into `env_root` will write a new Lemonade tree, and if so +/// what the user needs to be told. `None` when the tree there is already current +/// and has a llama.cpp backend, so the install only re-checks it. +pub(crate) fn plan_install_preflight( + paths: &AppPaths, + env_root: Option<&Path>, + reinstall: bool, + available_space: impl FnOnce(&Path) -> Option, +) -> Option { + let target_root = lemonade_root(paths, env_root); + let runtime_dir = runtime_dir_in(&target_root); + let extraction = needs_extraction( + reinstall, + installed_runtime_version(&runtime_dir).as_deref(), + rocm_deps::LEMONADE_VERSION, + lemond_path_in(&runtime_dir).is_file(), + ); + let backend_missing = !runtime_dir.join("bin").join("llamacpp").is_dir(); + if !extraction && !backend_missing { + return None; + } + let other_install = recorded_install_dir(paths) + .filter(|recorded| !runtime_path_is_same_or_inside(recorded, &target_root)); + let other_install_listed = other_install + .as_deref() + .is_some_and(|other| storage_lists(paths, other)); + // `prepare_embeddable` removes the whole runtime folder, backends included, + // before it extracts whenever `needs_extraction` says so, so that is the + // space the install frees first. + let replaced_bytes = if extraction && runtime_dir.exists() { + directory_bytes(&runtime_dir) + } else { + 0 + }; + let available = available_space(&target_root); + Some(InstallPreflight { + target_root, + other_install, + other_install_listed, + replaced_bytes, + available, + }) +} + +/// Total size of the files under `dir`, best effort: symlinks are not followed +/// and anything unreadable counts as zero, because this only sizes the space an +/// install is about to free. +fn directory_bytes(dir: &Path) -> u64 { + let mut total = 0_u64; + let mut stack = vec![dir.to_path_buf()]; + while let Some(current) = stack.pop() { + let Ok(entries) = std::fs::read_dir(¤t) else { + continue; + }; + for entry in entries.flatten() { + let Ok(metadata) = entry.metadata() else { + continue; + }; + if metadata.is_dir() && !metadata.is_symlink() { + stack.push(entry.path()); + } else { + total = total.saturating_add(metadata.len()); + } + } + } + total +} + +impl InstallPreflight { + /// The lines printed before the install starts. Every claim is derived from + /// the fields the install itself acts on: the target is where it writes, and + /// `other_install` is set only when the record points somewhere else. + pub(crate) fn notice_lines(&self) -> Vec { + let free = self + .available + .map(|bytes| format!(" ({} free on that drive)", format_bytes(bytes))) + .unwrap_or_default(); + let mut lines = vec![ + format!( + "Installing Lemonade {} into {}", + rocm_deps::LEMONADE_VERSION, + self.target_root.display() + ), + format!( + " With its llama.cpp ROCm backend this is a download of up to about {} and up to about {} of disk space{free}; a smaller backend, if Lemonade picks one for this GPU, needs less. It can take several minutes.", + download_text(), + disk_text(), + ), + ]; + if self.replaced_bytes > 0 { + lines.push(format!( + " The existing Lemonade install here ({}) is replaced first, and that space is counted as free.", + format_bytes(self.replaced_bytes) + )); + } + if let Some(other) = self.other_install.as_ref() { + let pointer = if self.other_install_listed { + " See `rocm storage`." + } else { + "" + }; + lines.push(format!( + " Lemonade is already installed at {}; that copy is not reused for this location.{pointer}", + other.display() + )); + } + lines + } + + /// Whether the drive can hold the install. + /// + /// Refuses only below [`MIN_INSTALL_BYTES`], the one figure that holds for + /// every backend. The ROCm backend download and the finished tree are upper + /// bounds, because the backend is not chosen until after the extraction this + /// runs before, so a host with less room than they need is warned and + /// allowed to try: refusing it could turn away a host whose smaller backend + /// would fit. + pub(crate) fn space_verdict(&self) -> SpaceVerdict { + const OPERATION: &str = "install Lemonade"; + let Some(free_now) = self.available else { + return SpaceVerdict::Fits; + }; + // The install deletes the existing runtime folder before it downloads, + // so its space counts. The warnings say so, so the figure shown is the + // one compared. + let available = free_now.saturating_add(self.replaced_bytes); + let room = if self.replaced_bytes > 0 { + format!( + "{} will be free on the drive holding {} once the existing install there is replaced", + format_bytes(available), + self.target_root.display() + ) + } else { + format!( + "only {} is free on the drive holding {}", + format_bytes(available), + self.target_root.display() + ) + }; + let minimum = rocm_core::with_margin(MIN_INSTALL_BYTES); + if available < minimum { + return SpaceVerdict::Refuse(format!( + "{} Nothing was downloaded.", + insufficient_space_message(OPERATION, &self.target_root, minimum, available) + )); + } + // The margin is working room for the comparison only. What is printed is + // the same figure the notice printed, so one run never shows two numbers + // for the same download. + if available < rocm_core::with_margin(FRESH_INSTALL_DOWNLOAD_BYTES) { + return SpaceVerdict::Tight(format!( + "Warning: {room}, and Lemonade's llama.cpp ROCm backend alone is a download of up to about {} (plus room to unpack it). The install will continue: if Lemonade picks a smaller backend for this GPU it may fit, otherwise it will fail partway through.", + download_text() + )); + } + if available < rocm_core::with_margin(FRESH_INSTALL_DISK_BYTES) { + return SpaceVerdict::Tight(format!( + "Warning: free disk space may be insufficient to install Lemonade's llama.cpp ROCm backend: up to about {} needed (plus room to work in), and {room}. The install will continue, but may fail partway through.", + disk_text() + )); + } + SpaceVerdict::Fits + } +} + +/// The ROCm backend's download as printed everywhere: by the notice and by both +/// warnings, so they cannot disagree. +fn download_text() -> String { + format_bytes(FRESH_INSTALL_DOWNLOAD_BYTES) +} + +/// The finished install's disk use as printed everywhere; see [`download_text`]. +fn disk_text() -> String { + format_bytes(FRESH_INSTALL_DISK_BYTES) +} + +/// Free space on the drive that will hold `path`, with the e2e-only override that +/// lets a black-box scenario reach the refusal without filling a real disk. +pub(crate) fn available_space_for_install(path: &Path) -> Option { + #[cfg(feature = "e2e-test-hooks")] + if let Some(bytes) = std::env::var_os(FREE_SPACE_TEST_ENV) + .and_then(|value| value.to_str().and_then(|value| value.parse::().ok())) + { + return Some(bytes); + } + rocm_core::available_space_for_path(path) +} + +/// Test-only override of the free space [`available_space_for_install`] reports. +#[cfg(feature = "e2e-test-hooks")] +const FREE_SPACE_TEST_ENV: &str = "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES"; + +/// The folder a Lemonade install for `env_root` lives in: `/lemonade`, +/// or `/engines/lemonade` when there is none. +pub fn install_tree_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { + lemonade_root(paths, env_root) +} + +/// The parts of a Lemonade tree that make up the install itself. +/// +/// That is the extracted runtime with its backends, the cached archive, and +/// leftover extraction scratch. Deliberately not the whole folder: under +/// `/engines/lemonade` the same folder also holds the install record, logs +/// and server state. +pub fn install_tree_dirs(tree_root: &Path) -> [PathBuf; 3] { + [ + runtime_dir_in(tree_root), + tree_root.join("downloads"), + tree_root.join("extract"), + ] +} + +/// What Lemonade's install record says, as far as the CLI can tell. +/// +/// Three states rather than an `Option`, because they call for different +/// reporting: a record that cannot be read is not the same as no record, and a +/// record whose folder no longer holds an install means nothing is in use. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InstallRecord { + /// No record: Lemonade has never been installed by ROCm CLI, or its record + /// was cleared. + NotRecorded, + /// A record file exists but could not be read or parsed. + Unreadable(String), + /// The record names `runtime_dir`; `holds_install` is whether the server + /// binary is still there. + Recorded { + runtime_dir: PathBuf, + holds_install: bool, + }, +} + +/// Read Lemonade's install record. +pub fn install_record(paths: &AppPaths) -> InstallRecord { + let record = manifest_path(paths); + match fs::read(&record) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => InstallRecord::NotRecorded, + Err(error) => InstallRecord::Unreadable(format!("{}: {error}", record.display())), + Ok(bytes) => { + match serde_json::from_slice::(&bytes) { + Ok(manifest) => InstallRecord::Recorded { + holds_install: lemond_path_in(&manifest.runtime_dir).is_file(), + runtime_dir: manifest.runtime_dir, + }, + Err(error) => InstallRecord::Unreadable(format!("{}: {error}", record.display())), + } + } + } +} + +/// The runtime folder the install record points at, if there is a readable one. +pub fn recorded_runtime_dir(paths: &AppPaths) -> Option { + match install_record(paths) { + InstallRecord::Recorded { runtime_dir, .. } => Some(runtime_dir), + InstallRecord::NotRecorded | InstallRecord::Unreadable(_) => None, + } +} + +/// The runtime folder Lemonade's record points at, only when it still holds an +/// install. +fn recorded_install_dir(paths: &AppPaths) -> Option { + match install_record(paths) { + InstallRecord::Recorded { + runtime_dir, + holds_install: true, + } => Some(runtime_dir), + InstallRecord::Recorded { .. } + | InstallRecord::NotRecorded + | InstallRecord::Unreadable(_) => None, + } +} + +/// The install roots of the ROCm installs registered with ROCm CLI, read from +/// the runtime registry the same way storage finds its runtimes. Best effort: an +/// unreadable entry is skipped. +fn registered_install_roots(paths: &AppPaths) -> Vec { + let registry = paths.data_dir.join("runtimes").join("registry"); + let Ok(entries) = fs::read_dir(registry) else { + return Vec::new(); + }; + entries + .flatten() + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json")) + .filter_map(|entry| fs::read(entry.path()).ok()) + .filter_map(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter_map(|record| record.get("install_root")?.as_str().map(PathBuf::from)) + .collect() +} + +/// Whether `rocm storage` lists the Lemonade install at `runtime_dir`: it lists +/// the data folder's tree and the tree inside each registered ROCm install, and +/// nothing else. Both use [`install_tree_root`], so the two cannot drift apart. +fn storage_lists(paths: &AppPaths, runtime_dir: &Path) -> bool { + runtime_path_is_same_or_inside(runtime_dir, &install_tree_root(paths, None)) + || registered_install_roots(paths).iter().any(|root| { + runtime_path_is_same_or_inside( + runtime_dir, + &install_tree_root(paths, Some(&root.join("engines"))), + ) + }) +} + +/// What [`forget_install_record_inside`] did, so a caller can report it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RecordCleanup { + /// The record does not name anything inside the removed folder, or there is + /// no record. Nothing to report. + NotAffected, + /// The record named this runtime folder and has been removed. + Cleared(PathBuf), + /// The record names a folder that was removed, but deleting the record + /// failed: Lemonade still points at files that are gone. + Failed { + runtime_dir: PathBuf, + record: PathBuf, + error: String, + }, + /// The record could not be read, so it could not be compared with the + /// removed folder. + Unreadable(String), +} + +/// Forget the install record when the tree it points at was inside `removed_root`, +/// which the caller has just deleted. +/// +/// Leaving the record would point Lemonade at files that no longer exist. Without +/// it Lemonade reports itself as not installed, which is the true state. Every +/// outcome that leaves the user with something to know is returned rather than +/// printed, so the uninstall report can carry it. +pub fn forget_install_record_inside(paths: &AppPaths, removed_root: &Path) -> RecordCleanup { + forget_install_record_inside_with(paths, removed_root, |record| fs::remove_file(record)) +} + +/// [`forget_install_record_inside`] with the deletion injectable, so the failure +/// outcome can be tested without a file the OS refuses to delete. +fn forget_install_record_inside_with( + paths: &AppPaths, + removed_root: &Path, + remove_record: impl FnOnce(&Path) -> std::io::Result<()>, +) -> RecordCleanup { + let runtime_dir = match install_record(paths) { + InstallRecord::NotRecorded => return RecordCleanup::NotAffected, + InstallRecord::Unreadable(error) => return RecordCleanup::Unreadable(error), + InstallRecord::Recorded { runtime_dir, .. } => runtime_dir, + }; + if !runtime_path_is_same_or_inside(&runtime_dir, removed_root) { + return RecordCleanup::NotAffected; + } + let record = manifest_path(paths); + match remove_record(&record) { + Ok(()) => RecordCleanup::Cleared(runtime_dir), + Err(error) => RecordCleanup::Failed { + runtime_dir, + record, + error: error.to_string(), + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::install::{LemonadeInstallManifest, write_manifest}; + + fn paths_in(root: &Path) -> AppPaths { + AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + } + } + + /// Plant an extracted tree the way `prepare_embeddable` leaves one: the + /// server binary, the version marker, and a llama.cpp backend folder. + fn plant_current_tree(tree_root: &Path) { + let runtime_dir = runtime_dir_in(tree_root); + fs::create_dir_all(runtime_dir.join("bin").join("llamacpp").join("rocm-stable")).unwrap(); + fs::write(lemond_path_in(&runtime_dir), b"").unwrap(); + crate::install::record_runtime_version(&runtime_dir, rocm_deps::LEMONADE_VERSION).unwrap(); + } + + fn record_install_at(paths: &AppPaths, runtime_dir: &Path) { + write_manifest( + paths, + &LemonadeInstallManifest { + env_id: "lemonade-embeddable-test".to_owned(), + version: rocm_deps::LEMONADE_VERSION.to_owned(), + runtime_dir: runtime_dir.to_path_buf(), + lemond: runtime_dir.join("lemond"), + lemonade: runtime_dir.join("lemonade"), + backend_recipe: "llamacpp".to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + }, + ) + .unwrap(); + } + + const GIB: u64 = 1024 * 1024 * 1024; + + /// The release-test sequence: Lemonade installed into the data folder, then + /// an SDK runtime activated, so the next install targets the runtime's own + /// folder. The notice must name the new folder and the copy it is not reusing. + #[test] + fn a_new_location_is_announced_with_the_install_it_does_not_reuse() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + record_install_at(&paths, &runtime_dir_in(&base)); + let runtime_engines = temp.path().join("runtimes").join("key").join("engines"); + + let preflight = + plan_install_preflight(&paths, Some(&runtime_engines), false, |_| Some(100 * GIB)) + .expect("a folder with no Lemonade in it needs a full install"); + + assert_eq!(preflight.target_root, runtime_engines.join("lemonade")); + assert_eq!(preflight.other_install, Some(runtime_dir_in(&base))); + assert!( + preflight.other_install_listed, + "the data folder's tree is listed" + ); + let notice = preflight.notice_lines().join("\n"); + assert!( + notice.contains(&preflight.target_root.display().to_string()), + "{notice}" + ); + assert!( + notice.contains( + "a download of up to about 4.4 GiB and up to about 25.0 GiB of disk space" + ), + "{notice}" + ); + assert!( + notice.contains("a smaller backend, if Lemonade picks one for this GPU, needs less"), + "{notice}" + ); + assert!(notice.contains("100.0 GiB free on that drive"), "{notice}"); + assert!( + notice.contains(&format!( + "Lemonade is already installed at {}; that copy is not reused", + runtime_dir_in(&base).display() + )), + "{notice}" + ); + assert_eq!(preflight.space_verdict(), SpaceVerdict::Fits); + } + + /// A current tree with a backend is not a new install: nothing to announce, + /// so a plain `rocm serve` on a ready host prints nothing extra. + #[test] + fn a_current_tree_is_not_announced() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + + assert_eq!(plan_install_preflight(&paths, None, false, |_| None), None); + // `--reinstall` wipes and re-extracts the same tree, so it is announced, + // and the record pointing into that same tree is not "another" install. + record_install_at(&paths, &runtime_dir_in(&base)); + let reinstall = plan_install_preflight(&paths, None, true, |_| None).unwrap(); + assert_eq!(reinstall.other_install, None); + assert!( + !reinstall + .notice_lines() + .join("\n") + .contains("already installed"), + "the tree being reinstalled must not be reported as a second copy" + ); + } + + #[test] + fn a_tree_without_a_backend_is_announced() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + fs::remove_dir_all(runtime_dir_in(&base).join("bin")).unwrap(); + + assert!(plan_install_preflight(&paths, None, false, |_| None).is_some()); + } + + /// A record whose folder lost its install (a runtime uninstalled by a build + /// that left the record behind) names nothing that is installed, so the + /// notice must not claim a copy "is already installed" or "not reused". + #[test] + fn a_record_naming_a_folder_with_no_install_is_not_announced_as_a_copy() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let gone = temp.path().join("runtimes").join("removed").join("engines"); + record_install_at( + &paths, + &runtime_dir_in(&install_tree_root(&paths, Some(&gone))), + ); + assert!( + matches!( + install_record(&paths), + InstallRecord::Recorded { + holds_install: false, + .. + } + ), + "the fixture must be a dangling record" + ); + + let preflight = plan_install_preflight(&paths, None, false, |_| Some(100 * GIB)) + .expect("an empty data folder needs a full install"); + + assert_eq!(preflight.other_install, None); + let notice = preflight.notice_lines().join("\n"); + assert!(!notice.contains("already installed"), "{notice}"); + assert!(!notice.contains("not reused"), "{notice}"); + } + + fn preflight_with(available: Option, replaced_bytes: u64) -> InstallPreflight { + InstallPreflight { + target_root: PathBuf::from("/data/engines/lemonade"), + other_install: None, + other_install_listed: false, + replaced_bytes, + available, + } + } + + /// Only the one figure that holds for every backend can refuse: the backend + /// is chosen after this runs, and a host Lemonade gives a smaller backend + /// must not be turned away for a download it does not need. + #[test] + fn only_space_below_the_smallest_install_refuses_and_less_than_the_rocm_backend_warns() { + const MIB: u64 = 1024 * 1024; + let preflight = |available| preflight_with(available, 0); + let SpaceVerdict::Refuse(message) = preflight(Some(2 * MIB)).space_verdict() else { + panic!("2 MiB cannot hold the Lemonade archive and its extracted files"); + }; + assert!( + message.contains("not enough free disk space to install Lemonade:"), + "{message}" + ); + assert!(message.contains("Nothing was downloaded."), "{message}"); + + // Room for Lemonade itself but not for the ROCm backend download: warned + // and allowed, because a smaller backend may still fit. + for available in [200 * MIB, 2 * GIB] { + let SpaceVerdict::Tight(warning) = preflight(Some(available)).space_verdict() else { + panic!("{available} bytes is below the ROCm backend download: warn, not refuse"); + }; + assert!( + warning.contains("if Lemonade picks a smaller backend for this GPU it may fit"), + "{warning}" + ); + } + // Room for the download but not the finished tree. + let SpaceVerdict::Tight(warning) = preflight(Some(10 * GIB)).space_verdict() else { + panic!("10 GiB is below the finished install: warn"); + }; + assert!(!warning.contains("smaller backend"), "{warning}"); + assert_eq!( + preflight(Some(40 * GIB)).space_verdict(), + SpaceVerdict::Fits + ); + assert_eq!(preflight(None).space_verdict(), SpaceVerdict::Fits); + } + + /// The notice and both warnings describe the same download and the same + /// disk use, so they must print the same figures. The margin is for the + /// comparison only: with it printed, the notice said 4.4 GiB and the warning + /// about 4.8 GiB for the same backend. + #[test] + fn the_notice_and_the_warnings_print_the_same_download_and_disk_figures() { + let notice = preflight_with(Some(100 * GIB), 0).notice_lines().join("\n"); + let SpaceVerdict::Tight(download_warning) = + preflight_with(Some(2 * GIB), 0).space_verdict() + else { + panic!("2 GiB is below the ROCm backend download"); + }; + let SpaceVerdict::Tight(disk_warning) = preflight_with(Some(10 * GIB), 0).space_verdict() + else { + panic!("10 GiB is below the finished install"); + }; + + // Derived from the constants, not retyped: if the notice or a warning + // ever prints a different number, one of these fails. + let download = format_bytes(FRESH_INSTALL_DOWNLOAD_BYTES); + let disk = format_bytes(FRESH_INSTALL_DISK_BYTES); + assert!( + notice.contains(&format!("a download of up to about {download}")), + "{notice}" + ); + assert!( + notice.contains(&format!("up to about {disk} of disk space")), + "{notice}" + ); + assert!( + download_warning.contains(&format!("a download of up to about {download}")), + "{download_warning}" + ); + assert!( + disk_warning.contains(&format!("up to about {disk} needed")), + "{disk_warning}" + ); + // And the margin-inflated values are not what is shown. + let inflated_download = format_bytes(rocm_core::with_margin(FRESH_INSTALL_DOWNLOAD_BYTES)); + let inflated_disk = format_bytes(rocm_core::with_margin(FRESH_INSTALL_DISK_BYTES)); + assert_ne!(inflated_download, download); + assert_ne!(inflated_disk, disk); + assert!( + !download_warning.contains(&inflated_download), + "{download_warning}" + ); + assert!(!disk_warning.contains(&inflated_disk), "{disk_warning}"); + // The figures themselves, so a constant change is a visible edit here too. + assert_eq!((download.as_str(), disk.as_str()), ("4.4 GiB", "25.0 GiB")); + } + + /// An install that replaces the existing runtime folder deletes it before + /// downloading (`prepare_embeddable` removes the whole folder, backends + /// included), so its space is free by then. The preflight sizes that folder, + /// counts it, and says so; an install that keeps the tree frees nothing. + #[test] + fn a_reinstall_counts_the_space_of_the_install_it_replaces() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + plant_current_tree(&tree); + let payload = runtime_dir_in(&tree) + .join("bin") + .join("llamacpp") + .join("rocm-stable"); + fs::write(payload.join("payload.bin"), vec![0_u8; 123_456]).unwrap(); + + let reinstall = plan_install_preflight(&paths, None, true, |_| Some(2 * GIB)).unwrap(); + assert!( + reinstall.replaced_bytes >= 123_456, + "{}", + reinstall.replaced_bytes + ); + let notice = reinstall.notice_lines().join("\n"); + assert!( + notice.contains(&format!( + "The existing Lemonade install here ({}) is replaced first, and that space is counted as free.", + format_bytes(reinstall.replaced_bytes) + )), + "{notice}" + ); + + // A current tree whose backend is missing is topped up, not wiped. + fs::remove_dir_all(runtime_dir_in(&tree).join("bin")).unwrap(); + let kept = plan_install_preflight(&paths, None, false, |_| Some(2 * GIB)).unwrap(); + assert_eq!(kept.replaced_bytes, 0); + assert!(!kept.notice_lines().join("\n").contains("replaced first")); + + // The verdict counts it: 2 GiB free warns for the ROCm backend, but the + // same drive with a 24 GiB install about to be deleted does not. + assert!(matches!( + preflight_with(Some(2 * GIB), 0).space_verdict(), + SpaceVerdict::Tight(_) + )); + assert_eq!( + preflight_with(Some(2 * GIB), 30 * GIB).space_verdict(), + SpaceVerdict::Fits + ); + // When it still does not fit, the warning shows the figure compared. + let SpaceVerdict::Tight(warning) = preflight_with(Some(GIB), 2 * GIB).space_verdict() + else { + panic!("3 GiB is below the ROCm backend download"); + }; + assert!( + warning.contains("3.0 GiB will be free on the drive holding /data/engines/lemonade once the existing install there is replaced"), + "{warning}" + ); + } + + /// The notice points at `rocm storage` only for a copy storage lists: the + /// data folder's tree and trees inside registered ROCm installs. A recorded + /// install anywhere else is announced without the pointer, which would be + /// false for it. + #[test] + fn the_storage_pointer_is_only_given_for_a_copy_storage_lists() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let target = temp.path().join("target").join("engines"); + let notice_for = |paths: &AppPaths| { + let preflight = plan_install_preflight(paths, Some(&target), false, |_| None).unwrap(); + ( + preflight.other_install_listed, + preflight.notice_lines().join("\n"), + ) + }; + + // 1. In the data folder: listed. + let base = install_tree_root(&paths, None); + plant_current_tree(&base); + record_install_at(&paths, &runtime_dir_in(&base)); + let (listed, notice) = notice_for(&paths); + assert!( + listed + && notice + .contains("that copy is not reused for this location. See `rocm storage`."), + "{notice}" + ); + + // 2. Inside a registered ROCm install: listed. + let registered = temp.path().join("runtimes").join("registered"); + let registry = paths.data_dir.join("runtimes").join("registry"); + fs::create_dir_all(®istry).unwrap(); + fs::write( + registry.join("registered.json"), + serde_json::json!({ "runtime_key": "registered", "install_root": registered }) + .to_string(), + ) + .unwrap(); + let in_runtime = install_tree_root(&paths, Some(®istered.join("engines"))); + plant_current_tree(&in_runtime); + record_install_at(&paths, &runtime_dir_in(&in_runtime)); + let (listed, notice) = notice_for(&paths); + assert!(listed && notice.contains("See `rocm storage`."), "{notice}"); + + // 3. Anywhere else: announced, not pointed at. + let elsewhere = + install_tree_root(&paths, Some(&temp.path().join("custom").join("engines"))); + plant_current_tree(&elsewhere); + record_install_at(&paths, &runtime_dir_in(&elsewhere)); + let (listed, notice) = notice_for(&paths); + assert!(!listed, "{notice}"); + assert!( + notice.contains(&format!( + "Lemonade is already installed at {}", + runtime_dir_in(&elsewhere).display() + )), + "{notice}" + ); + assert!(!notice.contains("rocm storage"), "{notice}"); + } + + /// A runtime uninstall deletes the runtime's folder, Lemonade tree included. + /// The record pointing into it is forgotten; one pointing elsewhere is kept. + #[test] + fn the_record_is_forgotten_only_when_its_tree_was_removed() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let runtime_root = temp.path().join("runtimes").join("key"); + let per_runtime = install_tree_root(&paths, Some(&runtime_root.join("engines"))); + record_install_at(&paths, &runtime_dir_in(&per_runtime)); + + let other_runtime = temp.path().join("runtimes").join("other"); + assert_eq!( + forget_install_record_inside(&paths, &other_runtime), + RecordCleanup::NotAffected + ); + assert!(recorded_runtime_dir(&paths).is_some()); + + assert_eq!( + forget_install_record_inside(&paths, &runtime_root), + RecordCleanup::Cleared(runtime_dir_in(&per_runtime)) + ); + assert_eq!(recorded_runtime_dir(&paths), None); + assert!(!manifest_path(&paths).exists()); + // Nothing recorded: nothing to forget. + assert_eq!( + forget_install_record_inside(&paths, &runtime_root), + RecordCleanup::NotAffected + ); + } + + /// The folder is already gone when the record is cleared, so a failed + /// deletion leaves Lemonade pointing at nothing. That has to come back to + /// the caller with the record's path, not vanish into a log line. + #[test] + fn a_record_that_cannot_be_deleted_is_reported_not_swallowed() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let runtime_root = temp.path().join("runtimes").join("key"); + let per_runtime = install_tree_root(&paths, Some(&runtime_root.join("engines"))); + record_install_at(&paths, &runtime_dir_in(&per_runtime)); + + let outcome = forget_install_record_inside_with(&paths, &runtime_root, |_| { + Err(std::io::Error::other("sharing violation")) + }); + + assert_eq!( + outcome, + RecordCleanup::Failed { + runtime_dir: runtime_dir_in(&per_runtime), + record: manifest_path(&paths), + error: "sharing violation".to_owned(), + } + ); + assert!(manifest_path(&paths).exists(), "the record was not deleted"); + + // The uninstall report tells the user to delete that record. Doing so must + // clear the condition: Lemonade is then simply not recorded. + fs::remove_file(manifest_path(&paths)).unwrap(); + assert_eq!(install_record(&paths), InstallRecord::NotRecorded); + assert_eq!(recorded_runtime_dir(&paths), None); + } + + /// An unparseable record is not "no record": it may well name the removed + /// folder, and the caller must be told it could not be checked. + #[test] + fn an_unreadable_record_is_distinct_from_no_record() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + assert_eq!(install_record(&paths), InstallRecord::NotRecorded); + + let record = manifest_path(&paths); + fs::create_dir_all(record.parent().unwrap()).unwrap(); + fs::write(&record, b"{ not json").unwrap(); + + let InstallRecord::Unreadable(error) = install_record(&paths) else { + panic!("a corrupt record must not read as NotRecorded"); + }; + assert!(error.contains("runtime.json"), "{error}"); + assert_eq!(recorded_runtime_dir(&paths), None); + assert!(matches!( + forget_install_record_inside(&paths, temp.path()), + RecordCleanup::Unreadable(_) + )); + assert!(record.exists(), "an unreadable record must be left alone"); + } + + /// A record whose runtime folder lost its server binary names an install + /// that is not there; `holds_install` is what lets the storage report say so. + #[test] + fn a_record_reports_whether_its_folder_still_holds_an_install() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + record_install_at(&paths, &runtime_dir_in(&tree)); + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir_in(&tree), + holds_install: false + } + ); + plant_current_tree(&tree); + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir_in(&tree), + holds_install: true + } + ); + } + + /// `rocm runtimes uninstall` tells a user whose install record cannot be read + /// to run a plain `rocm engines install lemonade`. That is only true if the + /// install path overwrites the record without reading it and keeps the + /// current tree, so this runs the real record-writing sequence + /// (`install_and_record`, with nothing to download) over a corrupt record and + /// asserts the state it leaves: a readable record naming an install that is + /// there, and the tree untouched (a `--reinstall` would have wiped it). + #[test] + fn a_plain_install_replaces_a_corrupt_record_and_keeps_the_current_tree() { + let temp = tempfile::tempdir().unwrap(); + let paths = paths_in(temp.path()); + let tree = install_tree_root(&paths, None); + plant_current_tree(&tree); + let runtime_dir = runtime_dir_in(&tree); + fs::write( + crate::install::lemonade_path_in(&runtime_dir), + b"fake lemonade", + ) + .unwrap(); + let sentinel = runtime_dir + .join("bin") + .join("llamacpp") + .join("backend.sentinel"); + fs::write(&sentinel, b"a backend already installed").unwrap(); + + let record = crate::install::manifest_path(&paths); + fs::create_dir_all(record.parent().unwrap()).unwrap(); + fs::write(&record, b"{ not json").unwrap(); + assert!(matches!( + install_record(&paths), + InstallRecord::Unreadable(_) + )); + + let archive = b"never parsed: the tree is current, so nothing is extracted"; + let sha256 = format!("{:x}", ::digest(archive)); + let (manifest, _) = crate::install_and_record( + &paths, + None, + false, + |paths, env_root, reinstall| { + crate::install::prepare_embeddable_with( + paths, + env_root, + reinstall, + rocm_deps::LEMONADE_VERSION, + &sha256, + |_, destination| { + fs::write(destination, archive)?; + Ok(()) + }, + ) + }, + |_, _| Ok(None), + ) + .expect("a plain install over a corrupt record succeeds"); + + assert_eq!( + install_record(&paths), + InstallRecord::Recorded { + runtime_dir: runtime_dir.clone(), + holds_install: true + }, + "the corrupt record was not replaced" + ); + assert_eq!(manifest.runtime_dir, runtime_dir); + assert!( + sentinel.is_file(), + "a plain install must keep the current tree" + ); + } +} diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index 5c67cbcad4..25728566d3 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -68,7 +68,7 @@ pub(crate) struct LemonadeInstallManifest { /// /// Pure so the upgrade case is testable without a real archive. `installed` /// is the version recorded for the tree already on disk, if any. -fn needs_extraction( +pub(crate) fn needs_extraction( reinstall: bool, installed: Option<&str>, wanted: &str, @@ -78,7 +78,7 @@ fn needs_extraction( } /// The Lemonade runtime version last extracted into `runtime_dir`, if any. -fn installed_runtime_version(runtime_dir: &Path) -> Option { +pub(crate) fn installed_runtime_version(runtime_dir: &Path) -> Option { fs::read_to_string(runtime_dir.join(RUNTIME_VERSION_MARKER)) .ok() .map(|s| s.trim().to_owned()) @@ -89,7 +89,7 @@ fn installed_runtime_version(runtime_dir: &Path) -> Option { /// part of the same wipe that removes the tree it describes — "marker present /// implies extraction completed" holds by construction, with no separate /// bookkeeping to keep in sync. -fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { +pub(crate) fn record_runtime_version(runtime_dir: &Path, version: &str) -> Result<()> { fs::write(runtime_dir.join(RUNTIME_VERSION_MARKER), version).with_context(|| { format!( "failed to record installed Lemonade runtime version under {}", @@ -117,7 +117,7 @@ pub(crate) fn prepare_embeddable( /// sha256, and the download step all injectable — the seam a test uses to /// exercise the real decision of which source `installed_version` trusts, /// rather than reimplementing that decision inline. -fn prepare_embeddable_with( +pub(crate) fn prepare_embeddable_with( paths: &AppPaths, env_root: Option<&Path>, reinstall: bool, @@ -195,18 +195,18 @@ where }) } -fn manifest_path(paths: &AppPaths) -> PathBuf { +pub(crate) fn manifest_path(paths: &AppPaths) -> PathBuf { paths.engine_install_manifest_path(ENGINE_NAME) } -fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { +pub(crate) fn lemonade_root(paths: &AppPaths, env_root: Option<&Path>) -> PathBuf { env_root.map_or_else( || paths.engine_dir(ENGINE_NAME), |root| normalize_runtime_path_for_host(root).join(ENGINE_NAME), ) } -fn runtime_dir_in(root: &Path) -> PathBuf { +pub(crate) fn runtime_dir_in(root: &Path) -> PathBuf { root.join("runtime") } @@ -451,11 +451,11 @@ impl PinFileBackup { } } -fn lemond_path_in(runtime_dir: &Path) -> PathBuf { +pub(crate) fn lemond_path_in(runtime_dir: &Path) -> PathBuf { runtime_dir.join(platform_binary_name("lemond")) } -fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { +pub(crate) fn lemonade_path_in(runtime_dir: &Path) -> PathBuf { runtime_dir.join(platform_binary_name("lemonade")) } diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index 48deb7d142..efa731264c 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -28,6 +28,7 @@ mod archive_props; mod backend_alignment; mod device_policy; mod direct_llama; +mod footprint; mod install; mod llamacpp_backend; mod process; @@ -41,6 +42,12 @@ mod test_support; // the crate root before the split, and `apps/rocm` depends on this crate's // lib target. pub use backend_alignment::LEMONADE_BACKEND_ALIGNMENT_DISABLED_ENV; +// The read-only view of Lemonade's install trees and record that `apps/rocm` +// needs for `rocm storage` and `rocm runtimes uninstall`; see `footprint`. +pub use footprint::{ + InstallRecord, RecordCleanup, forget_install_record_inside, install_record, install_tree_dirs, + install_tree_root, recorded_runtime_dir, +}; pub(crate) const ENGINE_NAME: &str = "lemonade"; pub(crate) const DEFAULT_HOST: &str = "127.0.0.1"; @@ -358,6 +365,35 @@ fn detect_response() -> DetectResponse { } } +/// The install proper: extract the embeddable, pick and align the llama.cpp +/// backend, then write the install record, replacing whatever was there. +/// +/// The record is written, never read, on this path: a corrupt record is +/// overwritten by the next plain install and the current tree is kept, which is +/// why that is what the CLI advises when it cannot read the record. The two +/// steps are parameters so a test can run this real sequence without +/// downloading anything or running `lemonade`. +fn install_and_record( + paths: &AppPaths, + env_root: Option<&std::path::Path>, + reinstall: bool, + prepare_embeddable: impl FnOnce( + &AppPaths, + Option<&std::path::Path>, + bool, + ) -> Result, + prepare_backend: impl FnOnce( + &AppPaths, + &mut crate::install::LemonadeInstallManifest, + ) -> Result>, +) -> Result<(crate::install::LemonadeInstallManifest, Option)> { + let mut manifest = prepare_embeddable(paths, env_root, reinstall)?; + eprintln!("Detecting best supported Lemonade llama.cpp backend..."); + let aligned_version = prepare_backend(paths, &mut manifest)?; + crate::install::write_manifest(paths, &manifest)?; + Ok((manifest, aligned_version)) +} + fn install_response(request: InstallRequest) -> Result { let paths = AppPaths::discover()?; paths.ensure()?; @@ -371,20 +407,38 @@ fn install_response(request: InstallRequest) -> Result { bail!("scripted Lemonade backend install failure") })?; } - eprintln!( - "Preparing Lemonade embeddable {}...", - rocm_deps::LEMONADE_VERSION - ); let env_root = request .env_root .as_deref() .map(normalize_runtime_path_for_host); - let mut manifest = - crate::install::prepare_embeddable(&paths, env_root.as_deref(), request.reinstall)?; - eprintln!("Detecting best supported Lemonade llama.cpp backend..."); - let aligned_version = - crate::backend_alignment::prepare_llamacpp_backend_for_active_rocm(&paths, &mut manifest)?; - crate::install::write_manifest(&paths, &manifest)?; + // Announced and space-checked before anything is downloaded: a fresh tree + // is a multi-gigabyte install, and `rocm serve` reaches this on its own. + if let Some(preflight) = crate::footprint::plan_install_preflight( + &paths, + env_root.as_deref(), + request.reinstall, + crate::footprint::available_space_for_install, + ) { + for line in preflight.notice_lines() { + eprintln!("{line}"); + } + match preflight.space_verdict() { + crate::footprint::SpaceVerdict::Fits => {} + crate::footprint::SpaceVerdict::Tight(warning) => eprintln!("{warning}"), + crate::footprint::SpaceVerdict::Refuse(message) => bail!(message), + } + } + eprintln!( + "Preparing Lemonade embeddable {}...", + rocm_deps::LEMONADE_VERSION + ); + let (manifest, aligned_version) = install_and_record( + &paths, + env_root.as_deref(), + request.reinstall, + crate::install::prepare_embeddable, + crate::backend_alignment::prepare_llamacpp_backend_for_active_rocm, + )?; let mut warnings = vec![ "Lemonade is installed as a rocm-cli managed embeddable runtime".to_owned(), format!( diff --git a/engines/lemonade/src/process.rs b/engines/lemonade/src/process.rs index e90c5cd25d..6d96eed836 100644 --- a/engines/lemonade/src/process.rs +++ b/engines/lemonade/src/process.rs @@ -159,6 +159,40 @@ pub(crate) fn hide_child_console_window(command: &mut ProcessCommand) { #[cfg(not(windows))] pub(crate) const fn hide_child_console_window(_command: &mut ProcessCommand) {} +/// Which of this process's standard output streams are a console. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OutputConsoles { + pub(crate) stdout: bool, + pub(crate) stderr: bool, +} + +impl OutputConsoles { + pub(crate) fn detect() -> Self { + use std::io::IsTerminal; + Self { + stdout: std::io::stdout().is_terminal(), + stderr: std::io::stderr().is_terminal(), + } + } + + /// Whether a child that inherits both streams may be started without a + /// console window. + /// + /// `CREATE_NO_WINDOW` starts the child with no console at all, and a child + /// with no console writes nothing to a console handle it inherited: run + /// from a real Windows console, `cmd /c echo`, a Python child and + /// `lemonade.exe --version` all printed their output without the flag and + /// printed nothing with it. That is how `lemonade backends install`'s + /// progress for a multi-gigabyte download disappeared from an interactive + /// `rocm serve` while still showing up in a redirected log. When either + /// stream is a console this process has one, so the child shares it and no + /// window opens; when neither is (a pipe, a file, or no console at all) the + /// flag is still applied, so a console-less parent never flashes a window. + pub(crate) const fn child_may_hide_its_console(self) -> bool { + !(self.stdout || self.stderr) + } +} + fn child_process_path(path: &Path) -> String { if runtime_is_windows() { windows_child_path(path) @@ -309,6 +343,39 @@ pub(crate) fn run_lemonade_backend_install( backend: &str, process_env: &LemonadeProcessEnvironment, ) -> Result<()> { + let mut command = backend_install_command( + manifest, + host, + port, + backend, + process_env, + OutputConsoles::detect(), + hide_child_console_window, + )?; + let status = command + .status() + .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; + if !status.success() { + bail!("Lemonade backend install failed with status {status}"); + } + Ok(()) +} + +/// The `lemonade backends install` command, ready to run. +/// +/// It inherits stdout and stderr: the backend download's progress is the child's +/// own output. `consoles` and `hide_window` are parameters so a test can assert +/// whether the window is hidden for each console state; the one production +/// caller passes the real streams and [`hide_child_console_window`]. +fn backend_install_command( + manifest: &LemonadeInstallManifest, + host: &str, + port: u16, + backend: &str, + process_env: &LemonadeProcessEnvironment, + consoles: OutputConsoles, + hide_window: impl FnOnce(&mut ProcessCommand), +) -> Result { let mut command = ProcessCommand::new(&manifest.lemonade); command .arg("--host") @@ -322,14 +389,10 @@ pub(crate) fn run_lemonade_backend_install( .stdout(Stdio::inherit()) .stderr(Stdio::inherit()); apply_lemonade_process_environment(&mut command, process_env)?; - hide_child_console_window(&mut command); - let status = command - .status() - .with_context(|| format!("failed to run {}", manifest.lemonade.display()))?; - if !status.success() { - bail!("Lemonade backend install failed with status {status}"); + if consoles.child_may_hide_its_console() { + hide_window(&mut command); } - Ok(()) + Ok(command) } fn lemonade_pull_args(host: &str, port: u16, checkpoint_ref: &str) -> Vec { @@ -644,6 +707,75 @@ mod tests { use rocm_engine_protocol::DevicePolicy; use serde_json::json; + fn test_manifest() -> LemonadeInstallManifest { + LemonadeInstallManifest { + env_id: "lemonade-embeddable-test".to_owned(), + version: "test".to_owned(), + runtime_dir: PathBuf::from("rt"), + lemond: PathBuf::from("rt").join("lemond"), + lemonade: PathBuf::from("rt").join("lemonade"), + backend_recipe: "llamacpp".to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + } + } + + /// Build the real backend-install command for a console state and report + /// whether the window was hidden, along with the command itself. + fn backend_install_for(consoles: OutputConsoles) -> (ProcessCommand, bool) { + let mut hidden = false; + let command = backend_install_command( + &test_manifest(), + "127.0.0.1", + 8123, + "rocm", + &LemonadeProcessEnvironment::default(), + consoles, + |_| hidden = true, + ) + .expect("the command builds"); + (command, hidden) + } + + /// The backend install's progress is the child's own output. Hiding its + /// console while that output is a console discards it (see + /// `OutputConsoles::child_may_hide_its_console`), so the window is hidden + /// only when neither stream is one. The command is the real one: a dropped + /// call to the hide step, a hide step run unconditionally, or a console + /// check that ignores one of the streams each turn a case below red. + #[test] + fn the_backend_install_hides_its_window_only_when_no_output_is_a_console() { + for (stdout, stderr, expect_hidden) in [ + (true, true, false), + (true, false, false), + (false, true, false), + (false, false, true), + ] { + let (command, hidden) = backend_install_for(OutputConsoles { stdout, stderr }); + assert_eq!( + hidden, expect_hidden, + "stdout console={stdout}, stderr console={stderr}" + ); + assert_eq!(command.get_program(), PathBuf::from("rt").join("lemonade")); + let args: Vec<_> = command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect(); + assert_eq!( + args, + [ + "--host", + "127.0.0.1", + "--port", + "8123", + "backends", + "install", + "llamacpp:rocm" + ] + ); + } + } + #[test] fn models_ready_accepts_direct_serve_and_gates_router_backend() { // Direct-serve (stock llama-server): no `recipe_options` → ready by name. diff --git a/tests/e2e-cucumber/features/lemonade_installs.feature b/tests/e2e-cucumber/features/lemonade_installs.feature new file mode 100644 index 0000000000..e55ad7fe16 --- /dev/null +++ b/tests/e2e-cucumber/features/lemonade_installs.feature @@ -0,0 +1,64 @@ +Feature: Lemonade installs on disk + + # Lemonade manages its own runtime, and where it installs depends on whether a + # ROCm runtime is active: into the data folder when none is, into that + # runtime's own folder when one is. So `rocm engines install lemonade` before + # `rocm install sdk`, then `rocm serve` after it, leaves two full copies of + # roughly 25 GiB each, and the second one used to arrive silently from + # `rocm serve` (ROCMAI-825). These scenarios cover what the user is now told + # about that: what an install needs before it starts, both copies in the disk + # report, and the install record when a runtime holding a copy is removed. + # + # A new feature file rather than appended scenarios: these are about one + # behaviour area, Lemonade's footprint on disk, which spans `engines install`, + # `storage report` and `runtimes uninstall`. + # + # None downloads anything or needs a GPU, so all run on the mock lane every PR. + # lemonade-installs-01 and -02 need the `rocm/e2e-test-hooks` build + # `cargo xtask e2e` makes, to report a nearly full drive without filling a + # real one. + + @id:lemonade-installs-refused-without-space + Scenario: lemonade-installs-01 - A Lemonade install the drive cannot hold is refused before downloading + Given the drive for a new Lemonade install cannot hold even Lemonade itself + When the user installs the Lemonade engine + Then the Lemonade install is refused before anything is downloaded + And the user is told what the Lemonade install needs and where it would go + + # `rocm serve` reaches the same install on its own, which is how the second + # copy used to appear without a word. The free-space override also stands in + # for the GPU the no-GPU lane does not have, as serve-18's failure hook does. + @id:lemonade-installs-serve-refused-without-space + Scenario: lemonade-installs-02 - Serving with Lemonade on a drive that cannot hold it is refused before downloading + Given the drive for a new Lemonade install cannot hold even Lemonade itself + When the user serves a model with Lemonade + Then the Lemonade install is refused before anything is downloaded + And the user is told what the Lemonade install needs and where it would go + + @id:lemonade-installs-storage-lists-both-copies + Scenario: lemonade-installs-03 - The disk report lists both Lemonade copies and flags the unused one + Given Lemonade was installed in the data folder and again inside a ROCm install + When the user asks what ROCm CLI is keeping on disk + Then the report lists both Lemonade installs with their sizes + And the report flags the data folder's Lemonade install as unused + + @id:lemonade-installs-runtime-uninstall-clears-record + Scenario: lemonade-installs-04 - Removing the ROCm install that held Lemonade clears its install record + Given Lemonade was installed in the data folder and again inside a ROCm install + When the user uninstalls the ROCm install holding Lemonade + Then the CLI says Lemonade's install was removed with that ROCm install + And Lemonade is no longer recorded as installed + + @id:lemonade-installs-dangling-record-keeps-the-survivor + Scenario: lemonade-installs-05 - A record naming a folder with no install does not make the surviving copy reclaimable + Given the ROCm install that held Lemonade is gone and only the data folder's copy is left + When the user asks what ROCm CLI is keeping on disk + Then the report does not call the surviving Lemonade copy unused or reclaimable + + @id:lemonade-installs-read-only-runtime-keeps-its-copy + Scenario: lemonade-installs-06 - Uninstalling a read-only ROCm install leaves its Lemonade copy, and the report says so + Given a read-only ROCm install holds a Lemonade copy + When the user asks what ROCm CLI is keeping on disk + Then the report does not say that copy is removed with its ROCm install + When the user uninstalls the ROCm install holding Lemonade + Then that Lemonade copy is still on disk diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index dee0ee502f..5e9818d305 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -32,6 +32,7 @@ mod e2e { pub mod engines_steps; pub mod examine_steps; pub mod host_guard; + pub mod lemonade_install_steps; pub mod lifecycle_steps; pub mod logs_steps; pub mod model_catalog_steps; diff --git a/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs new file mode 100644 index 0000000000..10a98f0295 --- /dev/null +++ b/tests/e2e-cucumber/tests/e2e/lemonade_install_steps.rs @@ -0,0 +1,402 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Steps for `lemonade_installs.feature`: what the user is told about the +//! Lemonade copies ROCm CLI puts on disk (ROCMAI-825). +//! +//! Black-box throughout. The two Lemonade trees and the install record are +//! planted as plain files in the layout the engine writes, so no scenario +//! downloads anything; the refusal scenario reaches the engine's free-space +//! check through the `rocm/e2e-test-hooks` override instead of a full drive. + +use std::path::PathBuf; + +use cucumber::{given, then, when}; + +use crate::E2eWorld; + +/// Registry key of the planted runtime. Production-shaped (slug, no `:`), so it +/// is also a valid registry file name on Windows. +const RUNTIME_KEY: &str = "release-tarball-gfx1151"; +/// Payload planted in each tree, chosen so the two render as distinct sizes. +const DATA_FOLDER_TREE_BYTES: usize = 3 * 1024 * 1024; +const RUNTIME_TREE_BYTES: usize = 5 * 1024 * 1024; +/// Free space the refusal scenarios report for the install drive: 1 MiB, below +/// what any Lemonade install needs (its archive and extracted files), so the +/// engine refuses before fetching anything. Room for that but not for the +/// multi-gigabyte ROCm backend only warns, and is unit-tested. +const NEARLY_FULL_BYTES: &str = "1048576"; + +fn data_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") +} + +/// Where Lemonade installs when no ROCm runtime is active. +fn data_folder_tree(world: &E2eWorld) -> PathBuf { + data_dir(world).join("engines").join("lemonade") +} + +fn runtime_install_root(world: &E2eWorld) -> PathBuf { + data_dir(world) + .join("runtimes") + .join("tarball") + .join(RUNTIME_KEY) +} + +/// Where Lemonade installs while the planted runtime is active. +fn runtime_tree(world: &E2eWorld) -> PathBuf { + runtime_install_root(world).join("engines").join("lemonade") +} + +/// Lemonade's install record, in the engine's own location. +fn install_record(world: &E2eWorld) -> PathBuf { + data_folder_tree(world) + .join("manifests") + .join("runtime.json") +} + +const fn lemond_name() -> &'static str { + if cfg!(windows) { + "lemond.exe" + } else { + "lemond" + } +} + +/// An extracted tree: the server binary plus a llama.cpp backend payload. +fn plant_lemonade_tree(tree: &std::path::Path, payload_bytes: usize) { + let runtime_dir = tree.join("runtime"); + let backend = runtime_dir.join("bin").join("llamacpp").join("rocm-stable"); + std::fs::create_dir_all(&backend).expect("failed to create backend dir"); + std::fs::write(runtime_dir.join(lemond_name()), b"").expect("failed to write lemond"); + std::fs::write(backend.join("payload.bin"), vec![0_u8; payload_bytes]) + .expect("failed to write backend payload"); +} + +fn stdout(world: &E2eWorld) -> &str { + world + .cli_output + .as_deref() + .expect("no CLI output captured - did the When step run?") +} + +fn combined_output(world: &E2eWorld) -> String { + format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or_default(), + world.cli_stderr.as_deref().unwrap_or_default() + ) +} + +#[given("the drive for a new Lemonade install cannot hold even Lemonade itself")] +async fn install_drive_nearly_full(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_LEMONADE_FREE_SPACE_BYTES", + NEARLY_FULL_BYTES.into(), + )); +} + +#[when("the user installs the Lemonade engine")] +async fn user_installs_lemonade(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["engines", "install", "lemonade", "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the Lemonade install is refused before anything is downloaded")] +async fn assert_lemonade_install_refused(world: &mut E2eWorld) { + let output = combined_output(world); + // Without the hook the engine sees the real drive and starts a real + // multi-gigabyte install. Name that cause rather than a baffling mismatch. + assert!( + output.contains("Installing Lemonade"), + "no install notice: the binary under test was probably built without the \ + `rocm/e2e-test-hooks` feature, which `cargo xtask e2e` enables:\n{output}" + ); + assert_ne!( + world.cli_rc, + Some(0), + "the install must fail when the drive cannot hold it:\n{output}" + ); + let tree = data_folder_tree(world); + // The claim "Nothing was downloaded" is checked against the disk: no + // archive cache and no extracted runtime. + assert!( + !tree.join("downloads").exists(), + "an archive was downloaded despite the refusal:\n{output}" + ); + assert!( + !tree.join("runtime").exists(), + "a runtime was extracted despite the refusal:\n{output}" + ); + assert!( + !install_record(world).exists(), + "a refused install must not record Lemonade as installed:\n{output}" + ); +} + +#[then("the user is told what the Lemonade install needs and where it would go")] +async fn assert_lemonade_install_notice(world: &mut E2eWorld) { + let output = combined_output(world); + let tree = data_folder_tree(world); + for expected in [ + format!("into {}", tree.display()), + "a download of up to about".to_owned(), + "of disk space (1.0 MiB free on that drive)".to_owned(), + "not enough free disk space to install Lemonade:".to_owned(), + "Nothing was downloaded.".to_owned(), + ] { + assert!( + output.contains(&expected), + "missing `{expected}`:\n{output}" + ); + } +} + +/// Register the planted runtime: a removable install ROCm CLI made itself +/// (registry entry plus the in-tree copy `runtimes uninstall` checks before +/// deleting the folder), or, with `read_only`, an external one that uninstall +/// only unregisters. +fn plant_runtime(world: &E2eWorld, read_only: bool) { + let install_root = runtime_install_root(world); + std::fs::create_dir_all(&install_root).expect("failed to create runtime root"); + std::fs::write(install_root.join("payload.txt"), "payload") + .expect("failed to write runtime payload"); + let manifest = serde_json::to_string_pretty(&serde_json::json!({ + "runtime_key": RUNTIME_KEY, + "runtime_id": "therock-release:gfx1151", + "channel": "release", + "format": "tarball", + "family": "gfx1151", + "family_source": "manual", + "version": "1.0.0", + "install_root": install_root, + "selected_artifact_url": format!("https://example.invalid/{RUNTIME_KEY}.tar.gz"), + "read_only": read_only, + "installed_at_unix_ms": 1_700_000_000_000u64, + })) + .expect("failed to serialize runtime manifest"); + let registry = data_dir(world).join("runtimes").join("registry"); + std::fs::create_dir_all(®istry).expect("failed to create registry"); + std::fs::write(registry.join(format!("{RUNTIME_KEY}.json")), &manifest) + .expect("failed to write registry entry"); + if !read_only { + std::fs::write(install_root.join(".rocm-cli-runtime.json"), &manifest) + .expect("failed to write in-tree manifest"); + } +} + +/// Write Lemonade's install record naming `runtime_dir`, whether or not +/// anything is there. +fn record_install_at(world: &E2eWorld, runtime_dir: &std::path::Path) { + let record = install_record(world); + std::fs::create_dir_all(record.parent().expect("record has a parent")) + .expect("failed to create record dir"); + std::fs::write( + &record, + serde_json::to_string_pretty(&serde_json::json!({ + "env_id": "lemonade-embeddable-e2e", + "version": "e2e", + "runtime_dir": runtime_dir, + "lemond": runtime_dir.join(lemond_name()), + "lemonade": runtime_dir.join("lemonade"), + "backend_recipe": "llamacpp", + "backend_name": "rocm", + "installed_at_unix_ms": 0, + })) + .expect("failed to serialize install record"), + ) + .expect("failed to write install record"); +} + +#[given("Lemonade was installed in the data folder and again inside a ROCm install")] +async fn plant_two_lemonade_installs(world: &mut E2eWorld) { + plant_runtime(world, false); + // `rocm engines install lemonade` before any runtime, then `rocm serve` + // with the runtime active: one tree each, the record naming the second. + plant_lemonade_tree(&data_folder_tree(world), DATA_FOLDER_TREE_BYTES); + plant_lemonade_tree(&runtime_tree(world), RUNTIME_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +#[given("the ROCm install that held Lemonade is gone and only the data folder's copy is left")] +async fn plant_surviving_data_folder_copy(world: &mut E2eWorld) { + // The record still names the removed runtime's folder, as it does after a + // runtime was uninstalled by a build that left the record behind. + plant_lemonade_tree(&data_folder_tree(world), DATA_FOLDER_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +#[given("a read-only ROCm install holds a Lemonade copy")] +async fn plant_lemonade_in_read_only_runtime(world: &mut E2eWorld) { + plant_runtime(world, true); + plant_lemonade_tree(&runtime_tree(world), RUNTIME_TREE_BYTES); + record_install_at(world, &runtime_tree(world).join("runtime")); +} + +/// The report row for the Lemonade tree at `tree`, and the lines under it. +fn lemonade_row<'a>(report: &'a str, tree: &std::path::Path) -> Vec<&'a str> { + let lines: Vec<&str> = report.lines().collect(); + let path = format!("({})", tree.display()); + let at = lines + .iter() + .position(|line| line.trim_start().starts_with("- lemonade (") && line.ends_with(&path)) + .unwrap_or_else(|| panic!("no Lemonade row for {}:\n{report}", tree.display())); + lines[at..] + .iter() + .take_while(|line| { + *line == &lines[at] + || line.trim_start().starts_with("status:") + || line.trim_start().starts_with("note:") + }) + .copied() + .collect() +} + +#[then("the report lists both Lemonade installs with their sizes")] +async fn assert_both_lemonade_rows(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row[0].contains("- lemonade (data folder): 3.0 MiB"), + "the data folder's copy must be listed with its size:\n{report}" + ); + let runtime_row = lemonade_row(&report, &runtime_tree(world)); + assert!( + runtime_row[0].contains(&format!( + "- lemonade (inside ROCm install {RUNTIME_KEY}): 5.0 MiB" + )), + "the runtime's copy must be listed with its size:\n{report}" + ); + assert!( + runtime_row + .iter() + .any(|line| line.trim() == "status: in use"), + "the copy the record points at is the one in use:\n{report}" + ); +} + +#[then("the report flags the data folder's Lemonade install as unused")] +async fn assert_data_folder_copy_unused(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row + .iter() + .any(|line| line.trim() + == "status: not used right now; Lemonade runs from another install"), + "the data folder's copy must be flagged unused:\n{report}" + ); + assert!( + data_row + .iter() + .any(|line| line.trim_start().starts_with("note: reclaimable")), + "an unused copy must say its space can be reclaimed:\n{report}" + ); + // Reporting only: the tree is still on disk after the report. + assert!(data_folder_tree(world).join("runtime").exists()); +} + +#[then("the report does not call the surviving Lemonade copy unused or reclaimable")] +async fn assert_survivor_not_called_reclaimable(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + row.iter().any(|line| line.trim() + == "status: not known to be used; the install record names a folder with no install"), + "a record naming a folder with no install leaves no copy known to be in use:\n{report}" + ); + assert!( + row.iter() + .all(|line| !line.contains("reclaimable") || line.contains("not marked reclaimable")), + "the only working copy must not be offered for deletion:\n{report}" + ); + assert!( + data_folder_tree(world).join("runtime").exists(), + "reporting must not delete anything" + ); +} + +#[then("the report does not say that copy is removed with its ROCm install")] +async fn assert_read_only_copy_not_removed_with_runtime(world: &mut E2eWorld) { + let report = stdout(world).to_owned(); + let row = lemonade_row(&report, &runtime_tree(world)); + assert!( + row.iter().all(|line| !line.contains("removed with it")), + "uninstalling an external ROCm install only unregisters it:\n{report}" + ); + assert!( + row.iter() + .any(|line| line + .contains("does not delete that install's folder (external or imported)")), + "the row must say why the copy outlives an uninstall:\n{report}" + ); +} + +#[then("that Lemonade copy is still on disk")] +async fn assert_lemonade_copy_still_on_disk(world: &mut E2eWorld) { + assert!( + runtime_tree(world).join("runtime").exists(), + "uninstalling an external ROCm install must leave its Lemonade copy" + ); +} + +#[when("the user uninstalls the ROCm install holding Lemonade")] +async fn user_uninstalls_runtime_holding_lemonade(world: &mut E2eWorld) { + let stdout = crate::run_rocm_ok(world, &["runtimes", "uninstall", RUNTIME_KEY, "--yes"]); + world.cli_output = Some(stdout); +} + +#[then("the CLI says Lemonade's install was removed with that ROCm install")] +async fn assert_lemonade_removed_with_runtime(world: &mut E2eWorld) { + let output = stdout(world).to_owned(); + let runtime_dir = runtime_tree(world).join("runtime"); + assert!( + output.contains(&format!( + "its install at {} is inside this runtime folder and would be removed with it", + runtime_dir.display() + )), + "the plan must warn that Lemonade goes with the runtime:\n{output}" + ); + assert!( + output.contains(&format!( + "its install at {} was inside this runtime and was removed with it", + runtime_dir.display() + )), + "the result must say Lemonade was removed with the runtime:\n{output}" + ); + assert!( + !runtime_install_root(world).exists(), + "the runtime folder, Lemonade tree included, must be gone:\n{output}" + ); +} + +#[then("Lemonade is no longer recorded as installed")] +async fn assert_lemonade_record_cleared(world: &mut E2eWorld) { + assert!( + !install_record(world).exists(), + "the record still names a deleted tree" + ); + // The CLI's own view agrees: the remaining copy is no longer called unused + // (that needs a record pointing elsewhere), and the deleted one is not listed. + let report = crate::run_rocm_ok(world, &["storage", "report"]); + let data_row = lemonade_row(&report, &data_folder_tree(world)); + assert!( + data_row + .iter() + .any(|line| line.trim() == "status: not recorded as installed"), + "with no record, the remaining copy is not recorded as installed:\n{report}" + ); + assert!( + !report.contains(&runtime_tree(world).display().to_string()), + "the removed copy must not be listed any more:\n{report}" + ); +} diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 230f791c8b..ab4ad13c8e 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -36,6 +36,9 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("engine_shell.feature", "engine-shell"), ("examine.feature", "examine"), ("install_lifecycle.feature", "lifecycle"), + // Not `lemonade`: the file is about the copies Lemonade leaves on disk, + // not about serving with it, which `model_serving.feature` covers. + ("lemonade_installs.feature", "lemonade-installs"), // Not `records`: a key has to say what kind of record, and this file is // about the ones a managed `rocm serve` leaves behind. ("local_server_records.feature", "server-records"),