diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..2151dc3607 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3741,7 +3741,6 @@ dependencies = [ "unicode-width", "ureq", "windows-native-keyring-store", - "windows-sys 0.61.2", "zbus-secret-service-keyring-store", ] @@ -3937,7 +3936,6 @@ dependencies = [ "sha2", "tokio", "ureq", - "windows-sys 0.61.2", ] [[package]] diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..7d53b8c7d4 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -65,7 +65,6 @@ libc.workspace = true [target.'cfg(windows)'.dependencies] self-replace = "1.5" windows-native-keyring-store = "1.1" -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1.0", features = ["keychain"] } diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 885e0bade4..731c1afee1 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -473,13 +473,9 @@ impl ActivationSnapshot { // failed, so it is the write least able to afford leaving a // truncated marker behind — which runtime resolution and the // storage retention holds would both read as "nothing is active". + // The helper creates the `runtimes` folder when it is missing. MarkerSnapshot::Contents(bytes) => { - let parent = path - .parent() - .context("active runtime marker path has no parent directory")?; - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - therock::write_file_atomically(&path, &bytes) + rocm_core::atomic_write::write_file_atomically(&path, &bytes) .with_context(|| format!("failed to restore {}", path.display()))?; } // Only a regular file is removed: whatever else may sit at that @@ -527,11 +523,11 @@ fn restore_after_failed_activation( error: anyhow::Error, ) -> anyhow::Error { match snapshot.restore(paths, config) { - Ok(()) => error.context( + Ok(()) => discard_failed_marker_replacement(error).context( "failed to record the active ROCm runtime; the previously active runtime is still \ the active one", ), - Err(restore_error) => error.context(format!( + Err(restore_error) => discard_failed_marker_replacement(error).context(format!( "failed to record the active ROCm runtime, and restoring the previous state failed \ too ({restore_error:#}); the ROCm CLI config and the active runtime marker may now \ disagree. Run `rocm runtimes list` to see both, then re-run `rocm runtimes activate \ @@ -540,6 +536,38 @@ fn restore_after_failed_activation( } } +/// Drop the marker a failed activation left behind, and the advice to use it. +/// +/// On Windows a marker publish can fail after taking the old marker away, and +/// the shared writer then keeps the new marker in its temp file and advises +/// renaming it into place (`rocm_core::atomic_write::KeptReplacement`). The +/// activation is being rolled back, so that file holds the marker of an +/// activation that failed, and renaming it would install that marker over +/// whatever the rollback restored. So the file is removed, and the error is +/// rebuilt without the advice. If the file cannot be removed, the error names +/// it and says to delete it instead. +fn discard_failed_marker_replacement(error: anyhow::Error) -> anyhow::Error { + let Some(kept) = error.downcast_ref::() else { + return error; + }; + let failure = format!( + "failed to write active runtime marker {}: {}", + kept.destination().display(), + kept.cause() + ); + match fs::remove_file(kept.kept()) { + Ok(()) => anyhow::anyhow!(failure), + Err(remove_error) if remove_error.kind() == std::io::ErrorKind::NotFound => { + anyhow::anyhow!(failure) + } + Err(remove_error) => anyhow::anyhow!( + "{failure}; {} holds the marker of the activation that failed and could not be \ + removed ({remove_error}): delete it, do not rename it into place", + kept.kept().display() + ), + } +} + /// Marker shown beside the active runtime in `rocm runtimes list`. Every /// place that renders this glyph MUST use this constant so the rendered /// character and the legend text stay in sync. @@ -843,12 +871,20 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { } } +/// Replace the active-runtime marker in one step. +/// +/// Two `rocm` processes can activate a runtime against the same data +/// directory at once (two terminals, a script), so the write goes through the +/// shared atomic writer: each writer stages into its own exclusively created +/// temp file, and the marker is replaced rather than removed first. On Unix, +/// and on Windows whenever `ReplaceFileW` succeeds, a reader therefore always +/// finds a complete marker. A `ReplaceFileW` that fails part-way is the +/// exception: it can leave the marker missing, which the writer repairs with a +/// plain rename when that rename succeeds (see `rocm_core::atomic_write`); two +/// replacements in flight at once on Windows are not tested. It does not order +/// the two activations: the last one to publish wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); - fs::create_dir_all( - path.parent() - .context("active runtime marker path has no parent directory")?, - )?; // The shared helper rather than a local temp-file-and-rename: it is the same // write the failed-activation restore uses, and it is strictly stronger than // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a @@ -858,13 +894,16 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> // window with no marker at all, which runtime resolution and the storage // retention holds both read as "nothing is active" — and a failed publish // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. - therock::write_file_atomically( - &path, - &serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, - ) - .with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) + // the folder `rocm runtimes` reads. The one exception is a Windows publish + // that took the old marker away and could not put the new one in place: + // the temp file is then the only copy, so it is kept and the error names + // it; a failed activation removes it again once the restore has put the + // previous marker back (`discard_failed_marker_replacement`). It also + // creates the `runtimes` folder when it is missing. + let bytes = + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; + rocm_core::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write active runtime marker {}", path.display())) } #[cfg(test)] @@ -1194,6 +1233,146 @@ pub(crate) mod tests { Ok(()) } + #[cfg(unix)] + fn test_active_runtime_marker(paths: &AppPaths, runtime_key: &str) -> ActiveRuntimeMarker { + ActiveRuntimeMarker { + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: runtime_key.to_owned(), + manifest_path: runtime_manifest_path(paths, runtime_key), + install_root: paths.data_dir.join("runtimes").join(runtime_key), + previous_runtime_id: None, + previous_runtime_key: None, + activated_at_unix_ms: rocm_core::unix_time_millis(), + } + } + + #[cfg(unix)] + fn leftover_marker_temp_files(paths: &AppPaths) -> Result> { + let dir = active_runtime_marker_path(paths) + .parent() + .context("marker has a parent")? + .to_path_buf(); + let mut leftovers = Vec::new(); + for entry in fs::read_dir(dir)? { + let name = entry?.file_name().to_string_lossy().into_owned(); + if name.starts_with("active.json.tmp-") { + leftovers.push(name); + } + } + Ok(leftovers) + } + + /// Two activations writing the marker at once must both succeed, and a + /// reader must find a complete marker throughout. #605 reported a writer + /// that removed `active.json` before renaming the replacement in, and two + /// writers in the same millisecond sharing one temp file, so one failed + /// with "failed to move active runtime marker" and the other could publish + /// the shorter payload followed by the longer one's tail. #424 fixed that; + /// this change routes the marker through `rocm_core::atomic_write`, the + /// writer `rocm` and `rocmd` now share, and this test pins that the marker + /// keeps the property through it. + /// + /// The bug is a race, so against the old code this fails with high + /// probability per round rather than deterministically; against the + /// current writer nothing in it is timing-dependent. Unix only: the + /// property under test is that `rename` replaces in one step. The Windows + /// publish path goes through `ReplaceFileW` instead, whose behaviour with + /// two replacements of one file in flight this test has not been run + /// against. + #[cfg(unix)] + #[test] + fn concurrent_active_runtime_marker_writes_keep_a_complete_marker_published() -> Result<()> { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Barrier}; + + const ROUNDS: usize = 200; + let (root, paths) = test_paths("marker-concurrent"); + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "initial"))?; + let marker_path = active_runtime_marker_path(&paths); + + let done = Arc::new(AtomicBool::new(false)); + let reader = { + let done = Arc::clone(&done); + let marker_path = marker_path.clone(); + std::thread::spawn(move || { + let mut problems = Vec::new(); + while !done.load(Ordering::Acquire) { + match fs::read(&marker_path) { + Ok(bytes) => { + if let Err(error) = + serde_json::from_slice::(&bytes) + { + problems.push(format!("unparsable marker: {error}")); + } + } + Err(error) => problems.push(format!("marker unreadable: {error}")), + } + } + problems + }) + }; + + let barrier = Arc::new(Barrier::new(2)); + let writers: Vec<_> = [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload", + ] + .into_iter() + .map(|runtime_key| { + let barrier = Arc::clone(&barrier); + let paths = paths.clone(); + std::thread::spawn(move || { + let mut errors = Vec::new(); + for _ in 0..ROUNDS { + barrier.wait(); + if let Err(error) = write_active_runtime_marker( + &paths, + test_active_runtime_marker(&paths, runtime_key), + ) { + errors.push(format!("{error:#}")); + } + } + errors + }) + }) + .collect(); + + let mut writer_errors = Vec::new(); + for writer in writers { + writer_errors.extend(writer.join().expect("writer thread panicked")); + } + done.store(true, Ordering::Release); + let reader_problems = reader.join().expect("reader thread panicked"); + + let published: ActiveRuntimeMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; + let leftovers = leftover_marker_temp_files(&paths)?; + let _ = fs::remove_dir_all(root); + + assert!( + writer_errors.is_empty(), + "{} of {} marker writes failed; first: {:?}", + writer_errors.len(), + 2 * ROUNDS, + writer_errors.first() + ); + assert!( + reader_problems.is_empty(), + "a reader found no complete marker {} times; first: {:?}", + reader_problems.len(), + reader_problems.first() + ); + assert!( + [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload" + ] + .contains(&published.runtime_key.as_str()), + "unexpected marker: {published:?}" + ); + assert_eq!(leftovers, Vec::::new()); + Ok(()) + } + #[test] fn runtime_activation_records_exact_key_and_rollback() -> Result<()> { let (root, paths) = test_paths("runtime-activation"); @@ -1299,10 +1478,10 @@ pub(crate) mod tests { } /// Replace the active-runtime marker with a NON-EMPTY directory, so the - /// next `write_active_runtime_marker` fails at a predictable point: its - /// `create_dir_all` of the parent succeeds, and the final `fs::rename` of - /// the temp file onto a non-empty directory fails on both supported - /// platforms without needing privileges or a real full disk. + /// next `write_active_runtime_marker` fails at a predictable point: the + /// temp file is staged beside it, and publishing that file onto a + /// non-empty directory fails on both supported platforms without needing + /// privileges or a real full disk. fn break_active_runtime_marker(paths: &AppPaths) -> Result { let marker = active_runtime_marker_path(paths); let _ = fs::remove_file(&marker); @@ -1994,6 +2173,198 @@ pub(crate) mod tests { Ok(()) } + /// The Windows partial-replace failure during activation: the marker publish + /// took the old marker away and kept the new one in its temp file. Once the + /// restore has put the previous marker back, the kept file holds the failed + /// activation's marker, so it is removed, and the error no longer tells the + /// user to rename it into place. Driven through the shared writer's publish + /// seam, so it runs on every platform. + #[test] + fn a_restored_activation_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.previous_runtime_key = Some(OLD_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + + let marker_path = active_runtime_marker_path(&paths); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!( + !marker_temp_file_names(&paths)?.is_empty(), + "the shared writer keeps the new marker when the old one is gone" + ); + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(&marker_path)?; + let leftovers = marker_temp_file_names(&paths)?; + let persisted = RocmCliConfig::load(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(restored_marker, previous_marker); + assert_eq!( + persisted.active_runtime_key.as_deref(), + Some(OLD_RUNTIME_KEY) + ); + assert!( + leftovers.is_empty(), + "the failed activation's marker was left beside the restored one: {leftovers:?}" + ); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// Produce the error the shared writer returns when a marker publish took + /// the old marker away and kept the new one, through its publish seam. + /// `prepare_kept` runs on the kept temp file before the error is returned. + fn partial_replace_marker_error( + paths: &AppPaths, + prepare_kept: impl FnOnce(&std::path::Path) -> std::io::Result<()>, + ) -> anyhow::Error { + let marker_path = active_runtime_marker_path(paths); + rocm_core::atomic_write::write_file_atomically_with_publish( + &marker_path, + b"{\"runtime_key\":\"new\"}", + |attempt| std::ffi::OsString::from(format!("partial-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path)?; + prepare_kept(tmp)?; + Err(rocm_core::atomic_write::destination_removed_error( + std::io::Error::from_raw_os_error(1176), + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + )) + }, + ) + .expect_err("the simulated partial replace fails the write") + .context(format!( + "failed to write active runtime marker {}", + marker_path.display() + )) + } + + /// When the kept marker cannot be removed, the error names it and says to + /// delete it, and nothing in it still says to rename it into place. The + /// kept path is made a non-empty directory, which `remove_file` refuses on + /// every platform. + #[test] + fn a_kept_marker_that_cannot_be_removed_is_named_with_delete_advice_only() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-stuck"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + let write_error = partial_replace_marker_error(&paths, |tmp| { + fs::remove_file(tmp)?; + fs::create_dir(tmp)?; + fs::write(tmp.join("occupied.txt"), "stuck") + }); + let kept = marker_temp_file_names(&paths)?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(active_runtime_marker_path(&paths))?; + let still_there = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert_eq!(still_there, kept, "the stuck file is left where it was"); + assert_eq!(restored_marker, previous_marker); + assert!( + rendered.contains(&kept[0]), + "the stuck file is named:\n{rendered}" + ); + assert!( + rendered.contains("delete it, do not rename it"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// A kept marker that is already gone by the time the rollback looks for it + /// is as good as removed: the error neither names it nor tells the user to + /// delete or rename it. + #[test] + fn a_kept_marker_already_gone_is_treated_as_removed() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-gone"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + let kept = marker_temp_file_names(&paths)?; + let runtimes_dir = active_runtime_marker_path(&paths) + .parent() + .context("active runtime marker path has no parent directory")? + .to_path_buf(); + for name in &kept { + fs::remove_file(runtimes_dir.join(name))?; + } + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!(!rendered.contains(&kept[0]), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(!rendered.contains("delete it"), "{rendered}"); + Ok(()) + } + + /// When the rollback itself fails as well, the kept marker is still the + /// failed activation's, so it goes too, and the error keeps the rollback's + /// own recovery advice without the advice to rename it. The config path is + /// made a non-empty directory so the restore's config save fails. + #[test] + fn a_failed_rollback_still_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-double-fault"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!(!marker_temp_file_names(&paths)?.is_empty()); + let config_path = paths.config_path(); + let _ = fs::remove_file(&config_path); + fs::create_dir_all(&config_path)?; + fs::write(config_path.join("occupied.txt"), "not a config")?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let leftovers = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert!( + rendered.contains("restoring the previous state failed too"), + "{rendered}" + ); + assert!(rendered.contains("rocm runtimes list"), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(leftovers.is_empty(), "{leftovers:?}"); + Ok(()) + } + // A restore that fails too must say so and name the way out, rather than // claim the previous runtime is still the active one. The marker's parent is // made a regular file, so writing the captured bytes back fails at diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 1cfe879e86..3371bbdb52 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -33,7 +33,6 @@ pub(crate) use self::consent::{ use self::family::{family_override_or_recovered_arch, resolve_family}; pub(crate) use self::fetch::truthy_env; use self::fetch::{download_file, preflight_tarball_space}; -pub(crate) use self::io::write_file_atomically; use self::probe::probe_rocm_sdk_runtime_for_target; pub(crate) use self::probe::{ RuntimeDeviceProbe, TorchAlignmentProbe, install_pinned_package, probe_rocm_sdk_runtime, @@ -1161,17 +1160,6 @@ mod tests { assert_eq!(estimate, archive * disk_space::EXTRACTED_SIZE_MULTIPLIER); } - #[test] - fn write_file_atomically_reports_a_full_disk_clearly() { - // Exercise the mapping the write path uses, without filling a disk. - let error = disk_space::map_write_error( - std::io::Error::from(std::io::ErrorKind::StorageFull), - Path::new("/cache/rocm.tar.gz.tmp"), - ); - let text = format!("{error:#}"); - assert!(text.contains("ran out of disk space"), "{text}"); - } - #[test] fn normalize_therock_family_maps_gfx1103_to_gfx110x_all() { assert_eq!( diff --git a/apps/rocm/src/therock/fetch.rs b/apps/rocm/src/therock/fetch.rs index 1257042bc0..6829709e25 100644 --- a/apps/rocm/src/therock/fetch.rs +++ b/apps/rocm/src/therock/fetch.rs @@ -8,14 +8,14 @@ //! //! Owns the pinned metadata trust root, //! which `cargo xtask verify-pinned-keys` checks against the canonical -//! published key. Depends on the sibling `io` module for atomic writes and on -//! `slugify` in the parent for cache file names. +//! published key. Depends on `rocm_core::atomic_write` for atomic writes and +//! on `slugify` in the parent for cache file names. -#[cfg(test)] -use super::io::{publish_temp_file, write_file_atomically_with_publish}; -use super::io::{stage_file_for_atomic_publish, write_file_atomically}; use super::slugify; use anyhow::{Context, Result, bail}; +#[cfg(test)] +use rocm_core::atomic_write::{publish_temp_file, write_file_atomically_with_publish}; +use rocm_core::atomic_write::{stage_file_for_atomic_publish, write_file_atomically}; use rocm_core::{AppPaths, disk_space, unix_time_millis, verify_rsa_pkcs1_sha256_signature}; use serde::{Deserialize, Serialize}; #[cfg(test)] diff --git a/apps/rocm/src/therock/io.rs b/apps/rocm/src/therock/io.rs index b4d3d06f72..9347220aee 100644 --- a/apps/rocm/src/therock/io.rs +++ b/apps/rocm/src/therock/io.rs @@ -2,21 +2,15 @@ // // SPDX-License-Identifier: MIT -//! Filesystem helpers behind TheRock installs: unique scratch directories and -//! the stage-then-publish atomic file writes. The writes serve the metadata -//! cache entries and the staged metadata public key, the startup update-check -//! record, and (through `therock`'s re-export of `write_file_atomically`) the -//! active-runtime marker that `runtimes_cmd` maintains. Runtime registry -//! manifests do not go through these writers. +//! Unique scratch directories behind TheRock installs. The atomic file writes +//! that used to live here are `rocm_core::atomic_write`, shared with `rocmd`. //! //! A leaf of the `therock` module: nothing here depends on the rest of it. use anyhow::{Context, Result, bail}; -use rocm_core::{disk_space, unix_time_millis}; -use std::ffi::{OsStr, OsString}; +use rocm_core::unix_time_millis; use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; pub(super) fn linux_temp_dir(prefix: &str) -> Result { let root = std::env::temp_dir(); @@ -69,456 +63,3 @@ fn windows_runtime_temp_root() -> Option { } None } - -/// A unique temp path next to `path`, preserving the full file name so a -/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes -/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -pub(crate) fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -pub(super) fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let tmp = stage_file_for_atomic_publish_with(path, bytes, suffix_for_attempt)?; - before_publish(); - publish_staged_file_with(&tmp, path, publish) -} - -pub(super) fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - stage_file_for_atomic_publish_with(path, bytes, |attempt| { - OsString::from(format!("{temp_id}-{attempt}")) - }) -} - -fn stage_file_for_atomic_publish_with( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, -) -> Result -where - S: FnMut(u32) -> OsString, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent)?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(disk_space::map_write_error(error, &tmp)); - } - drop(file); - Ok(tmp) -} - -#[cfg(test)] -fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { - publish_staged_file_with(tmp, path, publish_temp_file) -} - -fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> -where - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - publish(tmp, path) - .inspect_err(|_| { - let _ = fs::remove_file(tmp); - }) - .with_context(|| format!("failed to publish {}", path.display())) -} - -#[cfg(not(windows))] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn temp_sibling_path_preserves_multi_dot_file_names() { - let temp = temp_sibling_path( - Path::new("/tmp/cache/sdk.tar.gz"), - std::ffi::OsStr::new("test"), - ) - .unwrap(); - let name = temp.file_name().unwrap().to_string_lossy().into_owned(); - assert_eq!(name, "sdk.tar.gz.tmp-test"); - assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - std::ffi::OsString::from("same-millisecond") - } else { - std::ffi::OsString::from(format!( - "same-millisecond-{writer}-{attempt}" - )) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn concurrent_cached_publications_use_distinct_staging_files() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .map(|bytes| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - let staged = stage_file_for_atomic_publish(&destination, bytes)?; - before_publish.wait(); - publish_staged_file(&staged, &destination) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a cache writer must remain published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_string_lossy().contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - assert!( - leftovers.is_empty(), - "staged cache files leaked: {leftovers:?}" - ); - } - - #[test] - fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - fs::write(&destination, b"published").unwrap(); - let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); - - publish_staged_file_with(&staged, &destination, |_, _| { - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated cache publication failure", - )) - }) - .expect_err("simulated cache publication failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - assert!( - !staged.exists(), - "failed publication leaked its staging file" - ); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = std::ffi::OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("sdk.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. Run - /// with `cargo test -p rocm -- --ignored write_file_atomically_cleans_up`. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocm-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("artifact.tar.gz"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![0u8; 256 * 1024 * 1024]; - - let mut failures = Vec::new(); - for _ in 0..2 { - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - failures.push( - fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect::>(), - ); - } - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - for leftovers in &failures { - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - } - assert!( - !destination_exists, - "destination must not exist after failure" - ); - } -} diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 5d49245313..08060f6deb 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -23,7 +23,6 @@ //! install and activation paths share. use super::family::family_override_or_recovered_arch; -use super::io::write_file_atomically; use super::process::SuppressProgressOutput; use super::python::{ PythonLauncher, PythonSource, python_launcher_install_ready, python_requirement, @@ -41,6 +40,7 @@ use crate::runtime_registry::{ wheel_composition_device_target, }; use anyhow::{Context, Result, bail}; +use rocm_core::atomic_write::write_file_atomically; use rocm_core::{AppPaths, unix_time_millis}; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..945a47e903 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -24,6 +24,3 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } - -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 01b6d0ab71..9eaf6c8c4a 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -8,6 +8,7 @@ use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use crate::service::{stop_managed_service, stop_report_confirmed}; use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; +use rocm_core::atomic_write::write_file_atomically; use rocm_core::{ AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, ModelRecipeArtifactRecord, append_audit_event, model_artifact_cache_status, resolve_model_recipe_artifact, @@ -17,9 +18,8 @@ use serde_json::Value; use serde_json::json; #[cfg(test)] use sha2::Digest; -use std::ffi::{OsStr, OsString}; +#[cfg(any(test, target_os = "linux"))] use std::fs; -use std::io::{self, Write}; use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::{Command as ProcessCommand, Stdio}; @@ -667,160 +667,6 @@ fn sha256_hex(bytes: &[u8]) -> String { format!("{:x}", sha2::Sha256::digest(bytes)) } -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(error).with_context(|| format!("failed to write {}", tmp.display())); - } - drop(file); - before_publish(); - - publish(&tmp, path).inspect_err(|_| { - let _ = fs::remove_file(&tmp); - })?; - Ok(()) -} - -#[cfg(not(windows))] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - pub(crate) fn sandbox_check_updates_value(output: CommandCapture) -> Value { let status = update_check_status(&output); let update_available = @@ -997,7 +843,7 @@ mod tests { temp_app_paths, }; use rocm_core::ManagedServiceRecord; - use std::io::Read; + use std::io::{Read, Write}; fn serve_one_http_response(bytes: Vec) -> Result { let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; @@ -1017,198 +863,6 @@ mod tests { Ok(format!("http://{addr}/artifact.bin")) } - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("manifest.json"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-name-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("artifact.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - // Fails after the temp file exists, so the observed name is the real one. - write_file_atomically(&occupied, b"payload").expect_err("rename should fail"); - - let names: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!( - names, - vec!["artifact.tar.gz".to_owned()], - "only the occupied destination should remain" - ); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - OsString::from("same-millisecond") - } else { - OsString::from(format!("same-millisecond-{writer}-{attempt}")) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("manifest.json")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = OsString::from_vec(b"manifest-\xff.json".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocmd-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("manifest.json"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![b'x'; 256 * 1024 * 1024]; - - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - let leftovers: Vec = fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - assert!(!destination_exists); - } - #[test] fn sandbox_check_updates_value_is_read_only_and_preserves_output() { let value = sandbox_check_updates_value(CommandCapture { diff --git a/coverage-floors.toml b/coverage-floors.toml index aa2ece4edd..3cc2e5514e 100644 --- a/coverage-floors.toml +++ b/coverage-floors.toml @@ -23,5 +23,5 @@ "rocm-engine-lemonade" = 63.82 # 5554 lines "rocm-engine-protocol" = 98.37 # 491 lines "rocm-engine-vllm" = 66.76 # 3030 lines -"rocmd" = 75.36 # 7714 lines +"rocmd" = 75.02 # 7549 lines "xtask" = 76.18 # 5447 lines diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs new file mode 100644 index 0000000000..a3899fde64 --- /dev/null +++ b/crates/rocm-core/src/atomic_write.rs @@ -0,0 +1,1020 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Atomic file replacement, shared by `rocm` and `rocmd`. +//! +//! [`write_file_atomically`] stages the bytes in a fresh sibling temp file and +//! then publishes it over the destination in one step, so a reader sees either +//! the previous complete file or the new complete file, never a partial one, +//! and the destination is not removed before the new file takes its place. +//! +//! The temp file is named `{file name}.tmp-{pid}-{millis}-{attempt}` and +//! reserved with `create_new`, so two writers — threads of one process or +//! separate processes — can never write through the same temp file, even when +//! their pid and millisecond collide; a collision just moves on to the next +//! attempt. A failed write or publish removes its temp file, with the one +//! exception described below. +//! +//! On Windows, when the destination already exists, the publish uses +//! `ReplaceFileW` rather than `rename`. `std::fs::rename` does replace an +//! existing file there, but as `MoveFileExW` it fails with a sharing violation +//! while another process holds the destination open, which is the case +//! `ReplaceFileW` is used for. The concurrency tests below run on every +//! platform, but each starts with no destination, so on Windows at most one of +//! their publishes goes through `ReplaceFileW`: two `ReplaceFileW` publishes of +//! one file in flight at once are not exercised by any test. +//! +//! One failure leaves the destination absent: Microsoft documents that +//! `ReplaceFileW`, called without a backup name, can fail after it has already +//! moved the old file off the destination name +//! (`ERROR_UNABLE_TO_MOVE_REPLACEMENT` and `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`). +//! The publish then retries once as a plain rename. If that fails too, the +//! temp file is kept rather than removed, since it holds the only copy of the +//! new contents, and the error is a [`KeptReplacement`] naming it. The +//! decision rests on the error code the replace returned, not on whether the +//! destination existed beforehand, so a second writer that created the +//! destination in between cannot turn it into a deletion. +//! +//! Atomic, not durable: neither the temp file nor its directory is fsynced +//! before the publish. That survives a crashed process and a concurrent +//! writer, but after a power loss a filesystem that does not order data before +//! the rename can still show an empty or stale destination. +//! +//! This makes each file individually atomic. It does not serialize a +//! read-modify-write across processes: two writers that each read, change and +//! write the same file are still last-writer-wins. + +use crate::disk_space; +use crate::unix_time_millis; +use anyhow::{Context, Result, bail}; +use std::ffi::{OsStr, OsString}; +use std::fmt; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; + +/// The raw values of Windows' `ERROR_UNABLE_TO_MOVE_REPLACEMENT` (1176) and +/// `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` (1177), the two codes with which +/// `ReplaceFileW` reports that it moved the old file off the destination name +/// and then failed. Plain numbers so the decision built on them can be tested +/// on every platform; a Windows-only test ties them to `windows-sys`. +#[cfg(any(windows, test))] +const PARTIAL_REPLACE_ERROR_CODES: [i32; 2] = [1176, 1177]; + +/// Whether a failed `ReplaceFileW` left the destination name empty. +#[cfg(any(windows, test))] +fn is_partial_replace(error: &io::Error) -> bool { + error + .raw_os_error() + .is_some_and(|code| PARTIAL_REPLACE_ERROR_CODES.contains(&code)) +} + +/// A publish that took the old destination away and could not put the new +/// file in its place: the replace failed part-way, and so did the rename +/// retried after it. +#[derive(Debug)] +struct DestinationRemoved { + replace: io::Error, + retry: io::Error, +} + +impl fmt::Display for DestinationRemoved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}; retrying as a plain rename also failed: {}", + self.replace, self.retry + ) + } +} + +impl std::error::Error for DestinationRemoved {} + +/// Test seam: the error a publish returns when it took the old destination +/// away and could not put the new file in its place, which is what Windows' +/// partial `ReplaceFileW` followed by a failed rename produces. +/// +/// `pub` only so tests in other crates can drive that path through +/// [`write_file_atomically_with_publish`] on any platform; production code +/// never builds one. +#[doc(hidden)] +pub fn destination_removed_error(replace: io::Error, retry: io::Error) -> io::Error { + io::Error::new(retry.kind(), DestinationRemoved { replace, retry }) +} + +fn removed_the_destination(error: &io::Error) -> bool { + error + .get_ref() + .is_some_and(::is::) +} + +/// The error of a publish that took the old file away and kept the new +/// contents in the temp file, which then holds the only copy. +/// +/// A caller that knows better than to want those contents back — one that has +/// already put the previous file back, say — can downcast to this to find the +/// kept file and remove it. +#[derive(Debug)] +pub struct KeptReplacement { + destination: PathBuf, + kept: PathBuf, + cause: io::Error, +} + +impl KeptReplacement { + /// The path the publish was meant to replace, which is now missing. + pub fn destination(&self) -> &Path { + &self.destination + } + + /// The temp file holding the new contents. + pub fn kept(&self) -> &Path { + &self.kept + } + + /// Why the publish failed. + pub const fn cause(&self) -> &io::Error { + &self.cause + } +} + +impl fmt::Display for KeptReplacement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to publish {}, and the replace took the old file away; the new contents \ + are kept in {} - rename that file to {} to restore it", + self.destination.display(), + self.kept.display(), + self.destination.display() + ) + } +} + +impl std::error::Error for KeptReplacement { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.cause) + } +} + +/// What to do after `ReplaceFileW` failed with `error`: give up when the +/// destination is untouched, and otherwise put the replacement in place with +/// `rename`, which is a plain rename in production and injected by tests. +#[cfg(any(windows, test))] +fn recover_partial_replace( + error: io::Error, + replacement: &Path, + path: &Path, + rename: R, +) -> io::Result<()> +where + R: FnOnce(&Path, &Path) -> io::Result<()>, +{ + if !is_partial_replace(&error) { + return Err(error); + } + rename(replacement, path).map_err(|retry| destination_removed_error(error, retry)) +} + +/// A temp path next to `path`, preserving the full file name so a +/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes +/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). +fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { + let parent = path.parent().context("file path has no parent directory")?; + let mut file_name = path + .file_name() + .context("file path has no file name")? + .to_os_string(); + file_name.push(".tmp-"); + file_name.push(suffix); + Ok(parent.join(file_name)) +} + +fn process_temp_id() -> String { + format!("{}-{}", std::process::id(), unix_time_millis()) +} + +/// Replace `path` with `bytes` atomically, creating missing parent directories. +pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let temp_id = process_temp_id(); + write_file_atomically_with( + path, + bytes, + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + ) +} + +/// [`write_file_atomically`] with test seams for the temp name and publish point. +/// +/// `suffix_for_attempt` names the temp file for each attempt, and +/// `before_publish` runs after the temp file is fully written and before it is +/// published. Both exist so tests can force a temp-name collision and hold +/// writers at the publish point; production code calls +/// [`write_file_atomically`]. +fn write_file_atomically_with( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), +{ + write_file_atomically_with_publish( + path, + bytes, + suffix_for_attempt, + before_publish, + publish_temp_file, + ) +} + +/// Test seam: [`write_file_atomically`] with the temp name, a hook before the +/// publish, and the publish step itself injected. +/// +/// Tests use it to force a temp-name collision, to hold writers at the publish +/// point, to observe the staged file and the destination at the moment of +/// publication, or to make the publication fail. Pass [`publish_temp_file`] +/// for the real publish. Production code calls [`write_file_atomically`]. +/// +/// `pub` only because `apps/rocm`'s metadata-cache tests drive it across the +/// crate boundary; hidden from the docs because it is not an API to build on. +#[doc(hidden)] +pub fn write_file_atomically_with_publish( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, + publish: F, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + let tmp = stage_file_for_atomic_publish_with(path, bytes, suffix_for_attempt)?; + before_publish(); + publish_staged_file_with(&tmp, path, publish) +} + +/// Write `bytes` to a fresh, uniquely named sibling of `path` and return the +/// staged file's path without publishing it. +/// +/// The caller owns the staged file and must remove it once done with it; on a +/// staging failure nothing is left behind. To *replace* `path` with the bytes, +/// call [`write_file_atomically`] instead, which also removes the staged file +/// when the publish fails. +pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { + let temp_id = process_temp_id(); + stage_file_for_atomic_publish_with(path, bytes, |attempt| { + OsString::from(format!("{temp_id}-{attempt}")) + }) +} + +fn stage_file_for_atomic_publish_with( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, +) -> Result +where + S: FnMut(u32) -> OsString, +{ + stage_file_for_atomic_publish_with_writer(path, bytes, suffix_for_attempt, |file, bytes| { + file.write_all(bytes) + }) +} + +/// [`stage_file_for_atomic_publish_with`] with the write into the reserved +/// temp file injected, so a test can make it fail the way a full disk does +/// without filling one. +fn stage_file_for_atomic_publish_with_writer( + path: &Path, + bytes: &[u8], + mut suffix_for_attempt: S, + write: W, +) -> Result +where + S: FnMut(u32) -> OsString, + W: FnOnce(&mut fs::File, &[u8]) -> io::Result<()>, +{ + let parent = path.parent().context("file path has no parent directory")?; + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + + let mut reserved = None; + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + let suffix = suffix_for_attempt(attempt); + let tmp = temp_sibling_path(path, &suffix)?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + reserved = Some((tmp, file)); + break; + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", tmp.display())); + } + } + } + let Some((tmp, mut file)) = reserved else { + bail!( + "failed to reserve a temporary file next to {} after {} attempts", + path.display(), + ATOMIC_WRITE_TEMP_ATTEMPTS + ); + }; + + if let Err(error) = write(&mut file, bytes) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, path)); + } + drop(file); + Ok(tmp) +} + +#[cfg(test)] +fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { + publish_staged_file_with(tmp, path, publish_temp_file) +} + +/// Publish a staged file over `path`, removing the temp file if publishing +/// fails — unless the failed publish took the existing file away. +/// +/// A rename either replaces the destination or leaves it alone, but Windows' +/// `ReplaceFileW` can fail after moving the old file off the destination name +/// (see `recover_partial_replace`). Removing the temp file then would leave no +/// copy of the contents under any name the user can find, so it is kept and +/// the error is a [`KeptReplacement`] that says where. The publish step itself +/// reports that case, so it is told apart by the error rather than by a look +/// at the destination taken before the publish, which another writer can make +/// stale. +fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> +where + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + let Err(error) = publish(tmp, path) else { + return Ok(()); + }; + if removed_the_destination(&error) && tmp.exists() { + return Err(anyhow::Error::new(KeptReplacement { + destination: path.to_path_buf(), + kept: tmp.to_path_buf(), + cause: error, + })); + } + let _ = fs::remove_file(tmp); + Err(error).with_context(|| format!("failed to publish {}", path.display())) +} + +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. +#[cfg(not(windows))] +#[doc(hidden)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + fs::rename(tmp, path) +} + +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. +#[cfg(windows)] +#[doc(hidden)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + if path.try_exists()? { + return replace_file_windows(path, tmp); + } + + match fs::rename(tmp, path) { + Ok(()) => Ok(()), + Err(rename_error) => { + if path.try_exists()? { + replace_file_windows(path, tmp) + } else { + Err(rename_error) + } + } + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let replacement_wide: Vec = replacement + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + + // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and + // remain alive for the duration of the synchronous Windows API call. The + // optional backup, exclude, and reserved pointers are intentionally null. + let replaced = unsafe { + ReplaceFileW( + path_wide.as_ptr(), + replacement_wide.as_ptr(), + std::ptr::null(), + 0, + std::ptr::null(), + std::ptr::null(), + ) + }; + if replaced != 0 { + return Ok(()); + } + // Microsoft documents two partial outcomes for a call without a backup + // name, and in both the replacement is still under its own name: + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT` ("the replaced file no longer exists + // and the replacement file exists under its original name") and + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` ("the file to be replaced still + // exists with a different name"). Either way nothing is at `path` any + // more. One plain rename puts the new contents there; if that fails too, + // `publish_staged_file_with` keeps the replacement file. + recover_partial_replace(io::Error::last_os_error(), replacement, path, rename_file) +} + +/// `fs::rename` with both paths borrowed, the shape `recover_partial_replace` +/// takes. +#[cfg(windows)] +fn rename_file(from: &Path, to: &Path) -> io::Result<()> { + fs::rename(from, to) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_dir(label: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "rocm-core-{label}-{}-{}", + std::process::id(), + unix_time_millis() + )) + } + + /// A write that runs out of space while staging says so in words, and + /// leaves no temp file behind to keep the disk full. Drives the real + /// staging path — the one both [`write_file_atomically`] and + /// [`stage_file_for_atomic_publish`] go through — with only the byte write + /// into the reserved temp file swapped for one that fails like a full + /// disk, so the mapping is checked where it is applied rather than on its + /// own. + #[test] + fn staging_into_a_full_disk_reports_it_clearly_and_removes_the_temp_file() { + let root = scratch_dir("atomic-full-disk"); + let destination = root.join("rocm.tar.gz"); + + let error = stage_file_for_atomic_publish_with_writer( + &destination, + b"payload", + |attempt| OsString::from(format!("full-disk-{attempt}")), + |_, _| Err(io::Error::from(io::ErrorKind::StorageFull)), + ) + .expect_err("a write that hits a full disk must fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let _ = fs::remove_dir_all(&root); + let text = format!("{error:#}"); + assert!(text.contains("ran out of disk space"), "{text}"); + // The user knows the file they asked for, not the temp file that was + // already removed, so the message names the destination. + assert!( + text.contains(&destination.display().to_string()) && !text.contains(".tmp-"), + "{text}" + ); + assert!( + leftovers.is_empty(), + "the failed staging left files behind: {leftovers:?}" + ); + } + + /// When every temp name the writer tries is taken, it gives up after a + /// bounded number of tries with an error that says so, rather than + /// looping or writing through a file another writer reserved. + #[test] + fn staging_gives_up_when_every_temp_name_is_taken() { + let root = scratch_dir("atomic-names-exhausted"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("active.json"); + let taken = temp_sibling_path(&destination, OsStr::new("taken")).unwrap(); + fs::write(&taken, b"another writer's staged file").unwrap(); + let mut tries = 0; + + let error = stage_file_for_atomic_publish_with(&destination, b"payload", |_| { + tries += 1; + OsString::from("taken") + }) + .expect_err("a writer that finds every name taken must fail"); + + let foreign = fs::read(&taken); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + let text = format!("{error:#}"); + assert_eq!(tries, ATOMIC_WRITE_TEMP_ATTEMPTS); + assert!( + text.contains(&format!("after {ATOMIC_WRITE_TEMP_ATTEMPTS} attempts")), + "{text}" + ); + assert_eq!(foreign.unwrap(), b"another writer's staged file"); + assert_eq!(leftovers, vec![OsString::from("active.json.tmp-taken")]); + } + + /// The temp name keeps every extension, so a cleanup sweep over a cache + /// directory can still tell what a leftover was going to be. + #[test] + fn temp_sibling_path_preserves_multi_dot_file_names() { + let temp = + temp_sibling_path(Path::new("/tmp/cache/sdk.tar.gz"), OsStr::new("test")).unwrap(); + let name = temp.file_name().unwrap().to_string_lossy().into_owned(); + assert_eq!(name, "sdk.tar.gz.tmp-test"); + assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); + } + + /// Same property end to end: the temp file a write stages, observed at the + /// moment it is published, keeps the destination's full name, and the + /// publish leaves only the destination behind. + #[test] + fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { + let root = scratch_dir("atomic-name"); + let destination = root.join("artifact.tar.gz"); + let temp_id = process_temp_id(); + let mut staged_name = None; + + write_file_atomically_with_publish( + &destination, + b"payload", + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + |tmp, path| { + staged_name = tmp.file_name().map(OsStr::to_os_string); + publish_temp_file(tmp, path) + }, + ) + .unwrap(); + + let names: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let published = fs::read(&destination); + let _ = fs::remove_dir_all(&root); + assert_eq!( + staged_name, + Some(OsString::from(format!("artifact.tar.gz.tmp-{temp_id}-0"))) + ); + assert_eq!(published.unwrap(), b"payload"); + assert_eq!(names, vec!["artifact.tar.gz".to_owned()]); + } + + #[test] + fn concurrent_atomic_writes_do_not_remove_a_published_destination() { + let root = scratch_dir("atomic-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .enumerate() + .map(|(writer, bytes)| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + write_file_atomically_with( + &destination, + bytes, + |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + }, + || { + before_publish.wait(); + }, + ) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a writer must remain published"); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + } + + #[test] + fn concurrent_cached_publications_use_distinct_staging_files() { + let root = scratch_dir("cache-publish-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .enumerate() + .map(|(writer, bytes)| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + // Both threads ask for the same first name, as two writers + // sharing a pid and a millisecond would, so the collision + // happens every run rather than when the clock lines up. + let staged = + stage_file_for_atomic_publish_with(&destination, bytes, |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + })?; + before_publish.wait(); + publish_staged_file(&staged, &destination) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a cache writer must remain published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + assert!( + leftovers.is_empty(), + "staged cache files leaked: {leftovers:?}" + ); + } + + #[test] + fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { + let root = scratch_dir("cache-publish-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + fs::write(&destination, b"published").unwrap(); + let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); + + publish_staged_file_with(&staged, &destination, |_, _| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated cache publication failure", + )) + }) + .expect_err("simulated cache publication failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + assert!( + !staged.exists(), + "failed publication leaked its staging file" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn failed_atomic_replace_preserves_destination_and_cleans_temp() { + let root = scratch_dir("atomic-replace-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + fs::write(&destination, b"published").unwrap(); + + write_file_atomically_with_publish( + &destination, + b"replacement", + |attempt| OsString::from(format!("replace-failure-{attempt}")), + || {}, + |tmp, path| { + assert_eq!(fs::read(tmp).unwrap(), b"replacement"); + assert_eq!(path, destination); + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated atomic replacement failure", + )) + }, + ) + .expect_err("simulated replacement failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); + } + + fn simulated_partial_replace() -> io::Error { + destination_removed_error( + io::Error::from_raw_os_error(PARTIAL_REPLACE_ERROR_CODES[0]), + io::Error::from(io::ErrorKind::PermissionDenied), + ) + } + + /// A publish that fails after taking the old file away — what Windows' + /// `ReplaceFileW` documents for `ERROR_UNABLE_TO_MOVE_REPLACEMENT` without + /// a backup name, when the rename retried after it fails too — must not + /// also delete the temp file, which then holds the only copy. The error + /// names it so the user can put it back, and a caller can find it. + #[test] + fn a_publish_that_removes_the_destination_keeps_the_new_contents() { + let root = scratch_dir("replace-partial"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-{attempt}")), + || {}, + |_tmp, path| { + fs::remove_file(path).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + let message = format!("{error:#}"); + let reported = error + .downcast_ref::() + .map(|kept| (kept.kept().to_path_buf(), kept.destination().to_path_buf())); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let destination_exists = destination.exists(); + let _ = fs::remove_dir_all(&root); + assert!(!destination_exists); + assert_eq!(kept_bytes.unwrap(), b"new", "the new contents were deleted"); + assert!( + message.contains(&kept.display().to_string()), + "the error does not say where the contents are: {message}" + ); + assert_eq!(reported, Some((kept, destination))); + } + + /// The race a pre-publish look at the destination gets wrong: no + /// destination when this writer starts, another writer publishes one, and + /// then this writer's replace takes it away and fails. The temp file still + /// holds the only copy, so it is still kept. + #[test] + fn a_publish_that_removes_a_destination_another_writer_created_keeps_the_new_contents() { + let root = scratch_dir("replace-partial-race"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"second", + |attempt| OsString::from(format!("replace-partial-race-{attempt}")), + || {}, + |_tmp, path| { + fs::write(path, b"first").unwrap(); + fs::remove_file(path).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-race-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let _ = fs::remove_dir_all(&root); + assert_eq!( + kept_bytes.unwrap(), + b"second", + "the new contents were deleted" + ); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + } + + /// When the temp file is gone too there is nothing to keep, so the error + /// must not point the user at it. + #[test] + fn a_partial_replace_that_lost_the_temp_file_does_not_claim_to_keep_it() { + let root = scratch_dir("replace-partial-no-temp"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-no-temp-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path).unwrap(); + fs::remove_file(tmp).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + let _ = fs::remove_dir_all(&root); + let message = format!("{error:#}"); + assert!( + error.downcast_ref::().is_none(), + "{message}" + ); + assert!(!message.contains("kept in"), "{message}"); + } + + /// Any other failed replace leaves the destination alone, so nothing is + /// retried and the error comes back as it was. + #[test] + fn a_replace_that_left_the_destination_alone_is_not_retried() { + let mut renamed = false; + let error = recover_partial_replace( + io::Error::from_raw_os_error(5), + Path::new("staged"), + Path::new("destination"), + |_, _| { + renamed = true; + Ok(()) + }, + ) + .expect_err("an ordinary failure is returned"); + assert!(!renamed); + assert_eq!(error.raw_os_error(), Some(5)); + assert!(!removed_the_destination(&error)); + } + + /// Both documented partial codes retry once as a rename of the staged + /// file onto the destination, and a successful retry is a success. + #[test] + fn a_partial_replace_is_retried_as_a_rename() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let mut renamed = None; + recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |from, to| { + renamed = Some((from.to_path_buf(), to.to_path_buf())); + Ok(()) + }, + ) + .unwrap_or_else(|error| panic!("code {code}: {error}")); + assert_eq!( + renamed, + Some((PathBuf::from("staged"), PathBuf::from("destination"))), + "code {code}" + ); + } + } + + /// A retry that fails too is reported as the case that keeps the temp file. + #[test] + fn a_partial_replace_whose_retry_fails_reports_the_destination_removed() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let error = recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |_, _| Err(io::Error::from(io::ErrorKind::PermissionDenied)), + ) + .expect_err("a failed retry is returned"); + assert!(removed_the_destination(&error), "code {code}: {error}"); + assert!(error.to_string().contains("retrying as a plain rename")); + } + } + + /// The plain numbers the partial-replace decision uses are the codes + /// `ReplaceFileW` actually returns. + #[cfg(windows)] + #[test] + fn partial_replace_codes_match_the_windows_constants() { + use windows_sys::Win32::Foundation::{ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + }; + assert_eq!( + PARTIAL_REPLACE_ERROR_CODES.map(|code| u32::try_from(code).unwrap()), + [ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, + ERROR_UNABLE_TO_MOVE_REPLACEMENT_2 + ] + ); + } + + #[cfg(unix)] + #[test] + fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let file_name = OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); + let destination = Path::new("/tmp").join(&file_name); + let temp = temp_sibling_path(&destination, OsStr::new("collision")).unwrap(); + + let mut expected = file_name.into_vec(); + expected.extend_from_slice(b".tmp-collision"); + assert_eq!(temp.file_name().unwrap().as_bytes(), expected); + } + + /// Regression: a failed write must not leave a `.tmp-*` scratch file + /// behind. The name is unique per attempt, so before this an orphan + /// accumulated per retry — and when the failure is a full disk, those + /// orphans are exactly what keeps it full. + /// + /// Provokes the failure by pointing the destination at a non-empty + /// directory: the temp file is written, then neither the rename nor the + /// replace fallback can succeed. Portable, unlike an out-of-space test. + #[test] + fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { + let root = scratch_dir("atomic"); + let occupied = root.join("sdk.tar.gz"); + fs::create_dir_all(occupied.join("nested")).unwrap(); + fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); + + write_file_atomically(&occupied, b"payload") + .expect_err("renaming onto a non-empty directory should fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!( + leftovers.is_empty(), + "failed write left temp files behind: {leftovers:?}" + ); + } + + /// Mirrors the `/dev/shm` reproduction from the original report: a genuine + /// ENOSPC, not a rename failure standing in for one. + /// + /// Ignored by default because it fills `/dev/shm`, which is shared with + /// anything else on the host, so it is not safe to run concurrently. Run + /// with `cargo test -p rocm-core -- --ignored write_file_atomically_cleans_up`. + #[test] + #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] + fn write_file_atomically_cleans_up_temp_on_write_failure() { + let shm = Path::new("/dev/shm"); + if !shm.is_dir() { + eprintln!("skipping: /dev/shm unavailable"); + return; + } + let dir = shm.join(format!("rocm-core-enospc-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("artifact.tar.gz"); + // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. + let payload = vec![0u8; 256 * 1024 * 1024]; + + let mut failures = Vec::new(); + for _ in 0..2 { + write_file_atomically(&dest, &payload) + .expect_err("writing past the end of the filesystem should fail"); + failures.push( + fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(), + ); + } + let destination_exists = dest.exists(); + let _ = fs::remove_dir_all(&dir); + + for leftovers in &failures { + assert!( + leftovers.is_empty(), + "failed write left files behind: {leftovers:?}" + ); + } + assert!( + !destination_exists, + "destination must not exist after failure" + ); + } +} diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 58e59acfe4..62eaf05fad 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -1151,7 +1151,7 @@ last_installed_runtime_id = "therock-release" /// A directory at the destination is what makes the rename fail on both /// supported platforms without needing permissions a root CI lane ignores — /// the same device `write_file_atomically_cleans_up_temp_when_the_rename_fails` - /// uses in `apps/rocm`. + /// uses in `rocm_core::atomic_write`. #[test] fn a_config_save_that_cannot_publish_cleans_up_its_temp_file() -> Result<()> { let (root, paths) = temp_app_paths("config-save-rename-fails"); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aad2dda627..c8075f47a2 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -10,6 +10,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; pub mod amd_smi; #[cfg(all(test, unix))] mod archive_props; +pub mod atomic_write; pub mod automation; pub mod binary_paths; pub mod browser; diff --git a/docs/architecture.md b/docs/architecture.md index ad51f2aaf6..7d34b7e2f9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -103,9 +103,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and reads the os-release every one of these plans is built from through `rocm_core::hardware_root::real_os_release`, which has no copy here. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `rocm_core::hardware_root::real_os_release` and the three OpenMPI/libatomic/libnuma install hints ignore a simulated host root, and a scan that no source file of the binary reads os-release by hand. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. -- [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. +- [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`), both `pub(super)`. The atomic file writes that used to live here are rocm-core's shared [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), which [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs), [`therock/update.rs`](../apps/rocm/src/therock/update.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) call directly. - [`therock/process.rs`](../apps/rocm/src/therock/process.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for scratch directories (split out of `therock.rs`): running and capturing child commands (`run_command*`, `capture_command_output*`, `capture_python_stdout*`, `run_uv_progress_command`, and the Windows temp-file variants with `windows_child_path`), the bounded wait (`wait_with_output_bounded`), the progress-line output with its `--json` suppression guard (`SuppressProgressOutput`, `progress_line`), and unpacking the SDK tarball (`extract_tarball`, `extract_tarball_and_discard_archive`, which is why it sits here and not in `io.rs`: it spawns `tar` through `run_command`). Its `archive_properties` property tests run on Unix only. The process-wide environment lock these tests take is the single `PROCESS_ENV_TEST_LOCK` in [`test_support.rs`](../apps/rocm/src/test_support.rs), shared with the tests left in `therock.rs`. -- [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for atomic writes and on `slugify` in the parent for cache file names (split out of `therock.rs`): HTTP fetching for installs. `download_text_cached` is the metadata GET with its on-disk etag cache (`CachedHttp*`) and detached-signature verification (`MetadataSignaturePolicy`, `PINNED_METADATA_PUBLIC_KEY_PEM`, the unset trust root that `cargo xtask verify-pinned-keys` checks against the canonical published key, so its path is pinned in `xtask`); `download_file` is the streaming artifact download; `head_content_length` and `preflight_tarball_space` size the download and check free space first; `http_get` is the shared request. `truthy_env` is `pub(crate)` and re-exported by `therock.rs` for [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs); `download_text_cached`, `download_file`, `preflight_tarball_space` and the `CachedHttpText` struct (with its `text` field) that `download_text_cached` returns are `pub(super)` for the install and resolution code that stays in `therock.rs`; everything else is private. +- [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) — full domain extraction, a child of `therock.rs` that depends on rocm-core's [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) for atomic writes and on `slugify` in the parent for cache file names (split out of `therock.rs`): HTTP fetching for installs. `download_text_cached` is the metadata GET with its on-disk etag cache (`CachedHttp*`) and detached-signature verification (`MetadataSignaturePolicy`, `PINNED_METADATA_PUBLIC_KEY_PEM`, the unset trust root that `cargo xtask verify-pinned-keys` checks against the canonical published key, so its path is pinned in `xtask`); `download_file` is the streaming artifact download; `head_content_length` and `preflight_tarball_space` size the download and check free space first; `http_get` is the shared request. `truthy_env` is `pub(crate)` and re-exported by `therock.rs` for [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs); `download_text_cached`, `download_file`, `preflight_tarball_space` and the `CachedHttpText` struct (with its `text` field) that `download_text_cached` returns are `pub(super)` for the install and resolution code that stays in `therock.rs`; everything else is private. - [`therock/source.rs`](../apps/rocm/src/therock/source.rs) — full domain extraction, a child of `therock.rs` that depends only on `therock/version.rs` (split out of `therock.rs`): where an install reads from. `TheRockChannel` (release or nightly) and `SourceLayout` (the canonical stream, or ROCm 10's separately hosted next layout) are the two axes; `resolve_source` maps a pair to the `ResolvedSource` wheel-index and tarball-catalog bases, honouring a `ROCM_CLI_THEROCK_*_BASE` override only behind the explicit opt-in (`therock_base_override_allowed`, `select_base_override`, `env_override_base`); `select_source_layout`, `requested_source_layout` and `next_layout_requested` decide the layout from a pin; `RuntimeVersionSelector` is the version or build date a user pins (`pub(crate)`, re-exported by `therock.rs`). `FamilyResolution` lives here rather than in `therock/family.rs` because choosing the layout reads it; that keeps the dependency one-way, with `family.rs` building on this file. `TheRockChannel`, `SourceLayout`, `ResolvedSource` and `FamilyResolution` (with their methods and fields), `RuntimeVersionSelector`'s `describe` and `matches_version`, `resolve_source`, the three layout functions, and the `THEROCK_SOURCE_LAYOUT_GENERATION` and `THEROCK_NEXT_MIN_MAJOR` constants are `pub(super)` for `therock.rs` and its siblings; the base-override helpers and the other constants (the per-channel and next-layout base URLs, and the next layout's generation tag) are private to this file. - [`therock/family.rs`](../apps/rocm/src/therock/family.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` for `TheRockChannel` and `FamilyResolution` (split out of `therock.rs`): which package family a request targets. `resolve_family` picks it from `--family`, `ROCM_CLI_THEROCK_FAMILY`, a managed runtime or the host GPU, in that order, and refuses a GPU with no ROCm build (`unsupported_host_gpu_message`, `unsupported_requested_target_message`) instead of substituting another family; `family_override_or_recovered_arch` and `raw_arch_agreeing_with_family` recover the exact arch behind a grouped family; `family_resolution_hint` is the recovery text appended when no runtime is found. `resolve_family`, `family_override_or_recovered_arch` and `family_resolution_hint` are `pub(super)`; the rest is private. - [`therock/python.rs`](../apps/rocm/src/therock/python.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/io.rs`, `therock/process.rs` and `therock/source.rs` (split out of `therock.rs`): the Python an install runs on. `python_requirement` maps a `SourceLayout` to the interpreter the layout needs (`PythonRequirement`: wheel tag and `uv` version), which is why `source.rs` had to move first; `resolve_python_launcher` finds one in order of an explicit `ROCM_CLI_PYTHON`, `PATH`, the saved managed Python, then `ensure_managed_python`, which installs one through `uv` and records it in the managed-Python manifest; the resolver reads its environment through `PythonResolverEnv`, so tests steer it with a thread-local override (`OverridePythonResolverEnv`) rather than rewriting `PATH`; `wheel_compatibility_for_python` and `current_platform_wheel_tags` report the wheel tags an interpreter and platform accept (`WheelCompatibility`); `ensure_uv_venv` creates, or recreates when the interpreter tag no longer matches, the runtime's virtual environment. `venv_python_path` stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The items `therock.rs` reaches, and `split_runtime_path`, which `therock/probe.rs` reads, are `pub(super)`; the rest is private, including `ensure_managed_python` and `resolve_python_launcher_in` (the resolver's entry point that takes its environment as an argument). Its Unix-only tests drive fake interpreters and `uv` scripts. @@ -114,7 +114,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. - [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. -- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. +- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests and on rocm-core's [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) for the startup-check record (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. - [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK` and the background-helper fixtures `runtime_state` and `set_helper_facts`. - [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_serve_command_safety`, the four rules a model-proposed `serve` must satisfy (no public bind, no CPU device, a loopback host, `--managed` without `--foreground`), and `validate_chat_rocm_command_safety`, which applies them to a `serve` argv and also refuses a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_serve_command_safety` and `validate_chat_rocm_command_safety` here instead of through `chat`, and keeps no copy of the serve rules. `chat.rs`'s child modules call into it and `chat.rs` re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). - [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. The two modules depend on each other: `uninstall_plan.rs` reads this module's `pub(crate)` `PathEdit`, `PATH_EDIT_RECORD`, `plan_path_edits`, `plan_is_empty`, `removal_count` and `render_extra_plan`. @@ -150,7 +150,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). - [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. -- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. +- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. @@ -171,6 +171,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. +- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the shared atomic file writer `rocm` and `rocmd` both use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); atomic, not durable, since nothing is fsynced. Other hand-rolled temp-then-rename writers remain outside it, for example [`remote/session.rs`](../apps/rocm/src/remote/session.rs)'s session-file write, [`uninstall.rs`](../apps/rocm/src/uninstall.rs)'s profile replacement, `write_atomically` in [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs), and rocm-core's `write_atomic` in [`lib.rs`](../crates/rocm-core/src/lib.rs), which `RocmCliConfig::save` in [`config.rs`](../crates/rocm-core/src/config.rs), the automation event and proposal stores in [`automation.rs`](../crates/rocm-core/src/automation.rs), the JSONL stores in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and the service-record manifests in [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use; it is `pub` because `rocm-engine-lemonade` also writes through it, in [`install.rs`](../engines/lemonade/src/install.rs) (see the `write_atomic` entry below). #541, which builds on this writer, moves the config save onto it; #618 tracks that move. A new atomic write should call this module rather than add another copy. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. @@ -216,7 +217,7 @@ What else stays in [`lib.rs`](../crates/rocm-core/src/lib.rs), beside module dec - `sanitize_component` — the lowercase-alphanumeric slugger shared by [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs)'s `generate_service_id` and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs)'s artifact cache-marker names. It is not model-catalog-specific, so it does not move there. - `TORCH_ALIGNMENT_DISABLED_ENV`/`torch_alignment_disabled` — the one read of the torch-alignment opt-out, shared by the CLI and the vLLM engine so the two cannot disagree. - `VLLM_GPU_MEMORY_UTILIZATION_HINT` — a single wording contract between the CLI, the vLLM engine, and this crate's own `fix-16-vllm-oom` diagnosis; `rocm-core` is the only crate all of them can reach. -- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The open [#610](https://github.com/ROCm/rocm-cli/pull/610) adds a shared, stronger atomic writer to this crate for `rocm` and `rocmd`, and the open [#541](https://github.com/ROCm/rocm-cli/pull/541) moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). Once #610 lands, the automation and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. +- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The shared, stronger writer `rocm` and `rocmd` use is [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs); [#541](https://github.com/ROCm/rocm-cli/pull/541), which builds on it, moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). The automation, JSONL-store and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. Once the `write_atomic` move lands, [`lib.rs`](../crates/rocm-core/src/lib.rs) holds only module declarations, re-exports, the crate-root primitives above, the private `use` shims that let sibling modules reach some `host_gpu`, `managed_runtime`, `rocm_install` and `runtime` items as `crate::` paths, and its test-only modules — the end state of the modularization effort for this crate (ROCMAI-27, planned last: highest fan-in, since every app and engine crate depends on it, but lowest novelty, since the sibling modules already prove the pattern).