From 5f1701ff5ebfcc9f89fb02a936792cbc38a0f904 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 12:18:52 +0000 Subject: [PATCH 01/17] refactor(core): share one atomic file writer between rocm and rocmd `apps/rocm/src/therock.rs` and `apps/rocmd/src/sandbox.rs` each carried their own copy of `write_file_atomically` and its staging/publish helpers. Move the `therock` copy, the more complete of the two, into `rocm_core::atomic_write` and call it from both binaries, so the next caller (the active-runtime marker) gets the same implementation rather than a third copy. The move is mechanical apart from the error text. rocmd gets two extra pieces of context it was missing: the full-disk mapping through `disk_space::map_write_error`, and "failed to publish " when the rename fails. The `failed to create ` context that only rocmd had is kept. The helper tests move with the code. The five rocmd tests that duplicated therock tests are dropped. The one rocmd test that drove the temp-name check through a real write is kept. Count: 15 tests before (two of them ignored), 10 now (one ignored), with no assertion lost. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 4 +- apps/rocm/src/therock.rs | 12 - apps/rocm/src/therock/fetch.rs | 10 +- apps/rocm/src/therock/io.rs | 467 +----------------------- apps/rocm/src/therock/update.rs | 2 +- apps/rocmd/src/sandbox.rs | 351 +----------------- crates/rocm-core/src/atomic_write.rs | 522 +++++++++++++++++++++++++++ crates/rocm-core/src/config.rs | 2 +- crates/rocm-core/src/lib.rs | 1 + 9 files changed, 538 insertions(+), 833 deletions(-) create mode 100644 crates/rocm-core/src/atomic_write.rs diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 885e0bade4..29a7693588 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -479,7 +479,7 @@ impl ActivationSnapshot { .context("active runtime marker path has no parent directory")?; fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; - therock::write_file_atomically(&path, &bytes) + rocm_core::atomic_write::write_file_atomically(&path, &bytes) .with_context(|| format!("failed to restore {}", path.display()))?; } // Only a regular file is removed: whatever else may sit at that @@ -859,7 +859,7 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> // retention holds both read as "nothing is active" — and a failed publish // takes its temp file with it, so a retried activation cannot strand one in // the folder `rocm runtimes` reads. - therock::write_file_atomically( + rocm_core::atomic_write::write_file_atomically( &path, &serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, ) diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 1cfe879e86..3371bbdb52 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -33,7 +33,6 @@ pub(crate) use self::consent::{ use self::family::{family_override_or_recovered_arch, resolve_family}; pub(crate) use self::fetch::truthy_env; use self::fetch::{download_file, preflight_tarball_space}; -pub(crate) use self::io::write_file_atomically; use self::probe::probe_rocm_sdk_runtime_for_target; pub(crate) use self::probe::{ RuntimeDeviceProbe, TorchAlignmentProbe, install_pinned_package, probe_rocm_sdk_runtime, @@ -1161,17 +1160,6 @@ mod tests { assert_eq!(estimate, archive * disk_space::EXTRACTED_SIZE_MULTIPLIER); } - #[test] - fn write_file_atomically_reports_a_full_disk_clearly() { - // Exercise the mapping the write path uses, without filling a disk. - let error = disk_space::map_write_error( - std::io::Error::from(std::io::ErrorKind::StorageFull), - Path::new("/cache/rocm.tar.gz.tmp"), - ); - let text = format!("{error:#}"); - assert!(text.contains("ran out of disk space"), "{text}"); - } - #[test] fn normalize_therock_family_maps_gfx1103_to_gfx110x_all() { assert_eq!( diff --git a/apps/rocm/src/therock/fetch.rs b/apps/rocm/src/therock/fetch.rs index 1257042bc0..6829709e25 100644 --- a/apps/rocm/src/therock/fetch.rs +++ b/apps/rocm/src/therock/fetch.rs @@ -8,14 +8,14 @@ //! //! Owns the pinned metadata trust root, //! which `cargo xtask verify-pinned-keys` checks against the canonical -//! published key. Depends on the sibling `io` module for atomic writes and on -//! `slugify` in the parent for cache file names. +//! published key. Depends on `rocm_core::atomic_write` for atomic writes and +//! on `slugify` in the parent for cache file names. -#[cfg(test)] -use super::io::{publish_temp_file, write_file_atomically_with_publish}; -use super::io::{stage_file_for_atomic_publish, write_file_atomically}; use super::slugify; use anyhow::{Context, Result, bail}; +#[cfg(test)] +use rocm_core::atomic_write::{publish_temp_file, write_file_atomically_with_publish}; +use rocm_core::atomic_write::{stage_file_for_atomic_publish, write_file_atomically}; use rocm_core::{AppPaths, disk_space, unix_time_millis, verify_rsa_pkcs1_sha256_signature}; use serde::{Deserialize, Serialize}; #[cfg(test)] diff --git a/apps/rocm/src/therock/io.rs b/apps/rocm/src/therock/io.rs index b4d3d06f72..9347220aee 100644 --- a/apps/rocm/src/therock/io.rs +++ b/apps/rocm/src/therock/io.rs @@ -2,21 +2,15 @@ // // SPDX-License-Identifier: MIT -//! Filesystem helpers behind TheRock installs: unique scratch directories and -//! the stage-then-publish atomic file writes. The writes serve the metadata -//! cache entries and the staged metadata public key, the startup update-check -//! record, and (through `therock`'s re-export of `write_file_atomically`) the -//! active-runtime marker that `runtimes_cmd` maintains. Runtime registry -//! manifests do not go through these writers. +//! Unique scratch directories behind TheRock installs. The atomic file writes +//! that used to live here are `rocm_core::atomic_write`, shared with `rocmd`. //! //! A leaf of the `therock` module: nothing here depends on the rest of it. use anyhow::{Context, Result, bail}; -use rocm_core::{disk_space, unix_time_millis}; -use std::ffi::{OsStr, OsString}; +use rocm_core::unix_time_millis; use std::fs; -use std::io::{self, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; pub(super) fn linux_temp_dir(prefix: &str) -> Result { let root = std::env::temp_dir(); @@ -69,456 +63,3 @@ fn windows_runtime_temp_root() -> Option { } None } - -/// A unique temp path next to `path`, preserving the full file name so a -/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes -/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -pub(crate) fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -pub(super) fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let tmp = stage_file_for_atomic_publish_with(path, bytes, suffix_for_attempt)?; - before_publish(); - publish_staged_file_with(&tmp, path, publish) -} - -pub(super) fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - stage_file_for_atomic_publish_with(path, bytes, |attempt| { - OsString::from(format!("{temp_id}-{attempt}")) - }) -} - -fn stage_file_for_atomic_publish_with( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, -) -> Result -where - S: FnMut(u32) -> OsString, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent)?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(disk_space::map_write_error(error, &tmp)); - } - drop(file); - Ok(tmp) -} - -#[cfg(test)] -fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { - publish_staged_file_with(tmp, path, publish_temp_file) -} - -fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> -where - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - publish(tmp, path) - .inspect_err(|_| { - let _ = fs::remove_file(tmp); - }) - .with_context(|| format!("failed to publish {}", path.display())) -} - -#[cfg(not(windows))] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -pub(super) fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn temp_sibling_path_preserves_multi_dot_file_names() { - let temp = temp_sibling_path( - Path::new("/tmp/cache/sdk.tar.gz"), - std::ffi::OsStr::new("test"), - ) - .unwrap(); - let name = temp.file_name().unwrap().to_string_lossy().into_owned(); - assert_eq!(name, "sdk.tar.gz.tmp-test"); - assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - std::ffi::OsString::from("same-millisecond") - } else { - std::ffi::OsString::from(format!( - "same-millisecond-{writer}-{attempt}" - )) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn concurrent_cached_publications_use_distinct_staging_files() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .map(|bytes| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - let staged = stage_file_for_atomic_publish(&destination, bytes)?; - before_publish.wait(); - publish_staged_file(&staged, &destination) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a cache writer must remain published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .filter(|name| name.to_string_lossy().contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - assert!( - leftovers.is_empty(), - "staged cache files leaked: {leftovers:?}" - ); - } - - #[test] - fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { - let root = std::env::temp_dir().join(format!( - "rocm-cache-publish-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("index.body"); - fs::write(&destination, b"published").unwrap(); - let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); - - publish_staged_file_with(&staged, &destination, |_, _| { - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated cache publication failure", - )) - }) - .expect_err("simulated cache publication failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - assert!( - !staged.exists(), - "failed publication leaked its staging file" - ); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("sdk.tar.gz"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(std::io::Error::new( - std::io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = std::ffi::OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocm-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("sdk.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. Run - /// with `cargo test -p rocm -- --ignored write_file_atomically_cleans_up`. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocm-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("artifact.tar.gz"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![0u8; 256 * 1024 * 1024]; - - let mut failures = Vec::new(); - for _ in 0..2 { - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - failures.push( - fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect::>(), - ); - } - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - for leftovers in &failures { - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - } - assert!( - !destination_exists, - "destination must not exist after failure" - ); - } -} diff --git a/apps/rocm/src/therock/update.rs b/apps/rocm/src/therock/update.rs index 5d49245313..08060f6deb 100644 --- a/apps/rocm/src/therock/update.rs +++ b/apps/rocm/src/therock/update.rs @@ -23,7 +23,6 @@ //! install and activation paths share. use super::family::family_override_or_recovered_arch; -use super::io::write_file_atomically; use super::process::SuppressProgressOutput; use super::python::{ PythonLauncher, PythonSource, python_launcher_install_ready, python_requirement, @@ -41,6 +40,7 @@ use crate::runtime_registry::{ wheel_composition_device_target, }; use anyhow::{Context, Result, bail}; +use rocm_core::atomic_write::write_file_atomically; use rocm_core::{AppPaths, unix_time_millis}; use serde::{Deserialize, Serialize}; use std::cmp::Ordering; diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 01b6d0ab71..4bb1902939 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -8,6 +8,7 @@ use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use crate::service::{stop_managed_service, stop_report_confirmed}; use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; +use rocm_core::atomic_write::write_file_atomically; use rocm_core::{ AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, ModelRecipeArtifactRecord, append_audit_event, model_artifact_cache_status, resolve_model_recipe_artifact, @@ -17,9 +18,7 @@ use serde_json::Value; use serde_json::json; #[cfg(test)] use sha2::Digest; -use std::ffi::{OsStr, OsString}; use std::fs; -use std::io::{self, Write}; use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] use std::process::{Command as ProcessCommand, Stdio}; @@ -667,160 +666,6 @@ fn sha256_hex(bytes: &[u8]) -> String { format!("{:x}", sha2::Sha256::digest(bytes)) } -const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; - -fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { - let parent = path.parent().context("file path has no parent directory")?; - let mut file_name = path - .file_name() - .context("file path has no file name")? - .to_os_string(); - file_name.push(".tmp-"); - file_name.push(suffix); - Ok(parent.join(file_name)) -} - -fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { - let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); - write_file_atomically_with( - path, - bytes, - |attempt| OsString::from(format!("{temp_id}-{attempt}")), - || {}, - ) -} - -fn write_file_atomically_with( - path: &Path, - bytes: &[u8], - suffix_for_attempt: S, - before_publish: P, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), -{ - write_file_atomically_with_publish( - path, - bytes, - suffix_for_attempt, - before_publish, - publish_temp_file, - ) -} - -fn write_file_atomically_with_publish( - path: &Path, - bytes: &[u8], - mut suffix_for_attempt: S, - before_publish: P, - publish: F, -) -> Result<()> -where - S: FnMut(u32) -> OsString, - P: FnOnce(), - F: FnOnce(&Path, &Path) -> io::Result<()>, -{ - let parent = path.parent().context("file path has no parent directory")?; - fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; - - let mut reserved = None; - for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { - let suffix = suffix_for_attempt(attempt); - let tmp = temp_sibling_path(path, &suffix)?; - match fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&tmp) - { - Ok(file) => { - reserved = Some((tmp, file)); - break; - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} - Err(error) => { - return Err(error).with_context(|| format!("failed to create {}", tmp.display())); - } - } - } - let Some((tmp, mut file)) = reserved else { - bail!( - "failed to reserve a temporary file next to {} after {} attempts", - path.display(), - ATOMIC_WRITE_TEMP_ATTEMPTS - ); - }; - - if let Err(error) = file.write_all(bytes) { - drop(file); - let _ = fs::remove_file(&tmp); - return Err(error).with_context(|| format!("failed to write {}", tmp.display())); - } - drop(file); - before_publish(); - - publish(&tmp, path).inspect_err(|_| { - let _ = fs::remove_file(&tmp); - })?; - Ok(()) -} - -#[cfg(not(windows))] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - pub(crate) fn sandbox_check_updates_value(output: CommandCapture) -> Value { let status = update_check_status(&output); let update_available = @@ -997,7 +842,7 @@ mod tests { temp_app_paths, }; use rocm_core::ManagedServiceRecord; - use std::io::Read; + use std::io::{Read, Write}; fn serve_one_http_response(bytes: Vec) -> Result { let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; @@ -1017,198 +862,6 @@ mod tests { Ok(format!("http://{addr}/artifact.bin")) } - /// Regression: a failed write must not leave a `.tmp-*` scratch file - /// behind. The name is unique per attempt, so before this an orphan - /// accumulated per retry — and when the failure is a full disk, those - /// orphans are exactly what keeps it full. - /// - /// Provokes the failure by pointing the destination at a non-empty - /// directory: the temp file is written, then neither the rename nor the - /// replace fallback can succeed. Portable, unlike an out-of-space test. - #[test] - fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("manifest.json"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - write_file_atomically(&occupied, b"payload") - .expect_err("renaming onto a non-empty directory should fail"); - - let leftovers: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .filter(|name| name.contains(".tmp-")) - .collect(); - let _ = fs::remove_dir_all(&root); - assert!( - leftovers.is_empty(), - "failed write left temp files behind: {leftovers:?}" - ); - } - - /// The temp name keeps every extension, so a cleanup sweep over a cache - /// directory can still tell what a leftover was going to be. - #[test] - fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-name-{}-{}", - std::process::id(), - unix_time_millis() - )); - let occupied = root.join("artifact.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); - - // Fails after the temp file exists, so the observed name is the real one. - write_file_atomically(&occupied, b"payload").expect_err("rename should fail"); - - let names: Vec = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!( - names, - vec!["artifact.tar.gz".to_owned()], - "only the occupied destination should remain" - ); - } - - #[test] - fn concurrent_atomic_writes_do_not_remove_a_published_destination() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-collision-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); - - let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] - .into_iter() - .enumerate() - .map(|(writer, bytes)| { - let destination = destination.clone(); - let before_publish = std::sync::Arc::clone(&before_publish); - std::thread::spawn(move || { - write_file_atomically_with( - &destination, - bytes, - |attempt| { - if attempt == 0 { - OsString::from("same-millisecond") - } else { - OsString::from(format!("same-millisecond-{writer}-{attempt}")) - } - }, - || { - before_publish.wait(); - }, - ) - }) - }) - .collect(); - - for writer in writers { - writer.join().unwrap().unwrap(); - } - let published = fs::read(&destination).expect("a writer must remain published"); - let _ = fs::remove_dir_all(&root); - assert!(published == b"first" || published == b"second"); - } - - #[test] - fn failed_atomic_replace_preserves_destination_and_cleans_temp() { - let root = std::env::temp_dir().join(format!( - "rocmd-atomic-replace-failure-{}-{}", - std::process::id(), - unix_time_millis() - )); - fs::create_dir_all(&root).unwrap(); - let destination = root.join("manifest.json"); - fs::write(&destination, b"published").unwrap(); - - write_file_atomically_with_publish( - &destination, - b"replacement", - |attempt| OsString::from(format!("replace-failure-{attempt}")), - || {}, - |tmp, path| { - assert_eq!(fs::read(tmp).unwrap(), b"replacement"); - assert_eq!(path, destination); - Err(io::Error::new( - io::ErrorKind::PermissionDenied, - "simulated atomic replacement failure", - )) - }, - ) - .expect_err("simulated replacement failure must be returned"); - - assert_eq!(fs::read(&destination).unwrap(), b"published"); - let leftovers: Vec<_> = fs::read_dir(&root) - .unwrap() - .map(|entry| entry.unwrap().file_name()) - .collect(); - let _ = fs::remove_dir_all(&root); - assert_eq!(leftovers, vec![OsString::from("manifest.json")]); - } - - #[cfg(unix)] - #[test] - fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { - use std::os::unix::ffi::{OsStrExt, OsStringExt}; - - let file_name = OsString::from_vec(b"manifest-\xff.json".to_vec()); - let destination = Path::new("/tmp").join(&file_name); - let temp = temp_sibling_path(&destination, std::ffi::OsStr::new("collision")).unwrap(); - - let mut expected = file_name.into_vec(); - expected.extend_from_slice(b".tmp-collision"); - assert_eq!(temp.file_name().unwrap().as_bytes(), expected); - } - - /// Mirrors the `/dev/shm` reproduction from the original report: a genuine - /// ENOSPC, not a rename failure standing in for one. - /// - /// Ignored by default because it fills `/dev/shm`, which is shared with - /// anything else on the host, so it is not safe to run concurrently. - #[test] - #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] - fn write_file_atomically_cleans_up_temp_on_write_failure() { - let shm = Path::new("/dev/shm"); - if !shm.is_dir() { - eprintln!("skipping: /dev/shm unavailable"); - return; - } - let dir = shm.join(format!("rocmd-enospc-{}", std::process::id())); - let _ = fs::remove_dir_all(&dir); - fs::create_dir_all(&dir).unwrap(); - let dest = dir.join("manifest.json"); - // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. - let payload = vec![b'x'; 256 * 1024 * 1024]; - - write_file_atomically(&dest, &payload) - .expect_err("writing past the end of the filesystem should fail"); - let leftovers: Vec = fs::read_dir(&dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - let destination_exists = dest.exists(); - let _ = fs::remove_dir_all(&dir); - - assert!( - leftovers.is_empty(), - "failed write left files behind: {leftovers:?}" - ); - assert!(!destination_exists); - } - #[test] fn sandbox_check_updates_value_is_read_only_and_preserves_output() { let value = sandbox_check_updates_value(CommandCapture { diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs new file mode 100644 index 0000000000..5a19aca30b --- /dev/null +++ b/crates/rocm-core/src/atomic_write.rs @@ -0,0 +1,522 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Crash- and race-safe file replacement. +//! +//! [`write_file_atomically`] stages the bytes in a fresh sibling temp file and +//! then publishes it over the destination in one step, so a reader sees either +//! the previous complete file or the new complete file, never a partial one, +//! and the destination is never absent while it is being replaced. +//! +//! The temp file is named `{file name}.tmp-{pid}-{millis}-{attempt}` and +//! reserved with `create_new`, so two writers — threads of one process or +//! separate processes — can never write through the same temp file, even when +//! their pid and millisecond collide; a collision just moves on to the next +//! attempt. A failed write or publish removes its temp file. On Windows the +//! publish uses `ReplaceFileW` when the destination already exists, because +//! `rename` onto an existing file is not a replace there. +//! +//! This makes each file individually atomic. It does not serialize a +//! read-modify-write across processes: two writers that each read, change and +//! write the same file are still last-writer-wins. + +use crate::disk_space; +use crate::unix_time_millis; +use anyhow::{Context, Result, bail}; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; + +const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; + +/// A temp path next to `path`, preserving the full file name so a +/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes +/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). +fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { + let parent = path.parent().context("file path has no parent directory")?; + let mut file_name = path + .file_name() + .context("file path has no file name")? + .to_os_string(); + file_name.push(".tmp-"); + file_name.push(suffix); + Ok(parent.join(file_name)) +} + +fn process_temp_id() -> String { + format!("{}-{}", std::process::id(), unix_time_millis()) +} + +/// Replace `path` with `bytes` atomically, creating missing parent directories. +pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let temp_id = process_temp_id(); + write_file_atomically_with( + path, + bytes, + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + ) +} + +/// [`write_file_atomically`] with the temp-name suffix for each attempt and a +/// hook that runs after the temp file is fully written and before it is +/// published. Both exist so tests can force a temp-name collision and hold +/// writers at the publish point; production code calls +/// [`write_file_atomically`]. +pub fn write_file_atomically_with( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), +{ + write_file_atomically_with_publish( + path, + bytes, + suffix_for_attempt, + before_publish, + publish_temp_file, + ) +} + +/// [`write_file_atomically_with`] with the publish step injected as well, so +/// tests can observe the staged file and the destination at the moment of +/// publication, or make the publication fail. Pass [`publish_temp_file`] for +/// the real one. +pub fn write_file_atomically_with_publish( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, + before_publish: P, + publish: F, +) -> Result<()> +where + S: FnMut(u32) -> OsString, + P: FnOnce(), + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + let tmp = stage_file_for_atomic_publish_with(path, bytes, suffix_for_attempt)?; + before_publish(); + publish_staged_file_with(&tmp, path, publish) +} + +/// Write `bytes` to a fresh, uniquely named sibling of `path` and return the +/// staged file's path without publishing it. The caller owns the staged file: +/// it either publishes it or removes it. +pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { + let temp_id = process_temp_id(); + stage_file_for_atomic_publish_with(path, bytes, |attempt| { + OsString::from(format!("{temp_id}-{attempt}")) + }) +} + +fn stage_file_for_atomic_publish_with( + path: &Path, + bytes: &[u8], + mut suffix_for_attempt: S, +) -> Result +where + S: FnMut(u32) -> OsString, +{ + let parent = path.parent().context("file path has no parent directory")?; + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + + let mut reserved = None; + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + let suffix = suffix_for_attempt(attempt); + let tmp = temp_sibling_path(path, &suffix)?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + reserved = Some((tmp, file)); + break; + } + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", tmp.display())); + } + } + } + let Some((tmp, mut file)) = reserved else { + bail!( + "failed to reserve a temporary file next to {} after {} attempts", + path.display(), + ATOMIC_WRITE_TEMP_ATTEMPTS + ); + }; + + if let Err(error) = file.write_all(bytes) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, &tmp)); + } + drop(file); + Ok(tmp) +} + +#[cfg(test)] +fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { + publish_staged_file_with(tmp, path, publish_temp_file) +} + +fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> +where + F: FnOnce(&Path, &Path) -> io::Result<()>, +{ + publish(tmp, path) + .inspect_err(|_| { + let _ = fs::remove_file(tmp); + }) + .with_context(|| format!("failed to publish {}", path.display())) +} + +/// Move the staged file `tmp` over `path` in one step, replacing any existing +/// file without first removing it. +#[cfg(not(windows))] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + fs::rename(tmp, path) +} + +/// Move the staged file `tmp` over `path` in one step, replacing any existing +/// file without first removing it. +#[cfg(windows)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { + if path.try_exists()? { + return replace_file_windows(path, tmp); + } + + match fs::rename(tmp, path) { + Ok(()) => Ok(()), + Err(rename_error) => { + if path.try_exists()? { + replace_file_windows(path, tmp) + } else { + Err(rename_error) + } + } + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let replacement_wide: Vec = replacement + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + + // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and + // remain alive for the duration of the synchronous Windows API call. The + // optional backup, exclude, and reserved pointers are intentionally null. + let replaced = unsafe { + ReplaceFileW( + path_wide.as_ptr(), + replacement_wide.as_ptr(), + std::ptr::null(), + 0, + std::ptr::null(), + std::ptr::null(), + ) + }; + if replaced == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch_dir(label: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "rocm-core-{label}-{}-{}", + std::process::id(), + unix_time_millis() + )) + } + + #[test] + fn write_file_atomically_reports_a_full_disk_clearly() { + // Exercise the mapping the write path uses, without filling a disk. + let error = disk_space::map_write_error( + std::io::Error::from(std::io::ErrorKind::StorageFull), + Path::new("/cache/rocm.tar.gz.tmp"), + ); + let text = format!("{error:#}"); + assert!(text.contains("ran out of disk space"), "{text}"); + } + + /// The temp name keeps every extension, so a cleanup sweep over a cache + /// directory can still tell what a leftover was going to be. + #[test] + fn temp_sibling_path_preserves_multi_dot_file_names() { + let temp = + temp_sibling_path(Path::new("/tmp/cache/sdk.tar.gz"), OsStr::new("test")).unwrap(); + let name = temp.file_name().unwrap().to_string_lossy().into_owned(); + assert_eq!(name, "sdk.tar.gz.tmp-test"); + assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); + } + + /// Same property end to end: the temp file the real write creates (and + /// then cleans up) leaves only the destination behind. + #[test] + fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { + let root = scratch_dir("atomic-name"); + let occupied = root.join("artifact.tar.gz"); + fs::create_dir_all(occupied.join("nested")).unwrap(); + fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); + + // Fails after the temp file exists, so the observed name is the real one. + write_file_atomically(&occupied, b"payload").expect_err("rename should fail"); + + let names: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let _ = fs::remove_dir_all(&root); + assert_eq!( + names, + vec!["artifact.tar.gz".to_owned()], + "only the occupied destination should remain" + ); + } + + #[test] + fn concurrent_atomic_writes_do_not_remove_a_published_destination() { + let root = scratch_dir("atomic-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .enumerate() + .map(|(writer, bytes)| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + write_file_atomically_with( + &destination, + bytes, + |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + }, + || { + before_publish.wait(); + }, + ) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a writer must remain published"); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + } + + #[test] + fn concurrent_cached_publications_use_distinct_staging_files() { + let root = scratch_dir("cache-publish-collision"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + let before_publish = std::sync::Arc::new(std::sync::Barrier::new(2)); + + let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .map(|bytes| { + let destination = destination.clone(); + let before_publish = std::sync::Arc::clone(&before_publish); + std::thread::spawn(move || { + let staged = stage_file_for_atomic_publish(&destination, bytes)?; + before_publish.wait(); + publish_staged_file(&staged, &destination) + }) + }) + .collect(); + + for writer in writers { + writer.join().unwrap().unwrap(); + } + let published = fs::read(&destination).expect("a cache writer must remain published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .filter(|name| name.to_string_lossy().contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!(published == b"first" || published == b"second"); + assert!( + leftovers.is_empty(), + "staged cache files leaked: {leftovers:?}" + ); + } + + #[test] + fn failed_cached_publication_preserves_destination_and_cleans_staging_file() { + let root = scratch_dir("cache-publish-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("index.body"); + fs::write(&destination, b"published").unwrap(); + let staged = stage_file_for_atomic_publish(&destination, b"replacement").unwrap(); + + publish_staged_file_with(&staged, &destination, |_, _| { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated cache publication failure", + )) + }) + .expect_err("simulated cache publication failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + assert!( + !staged.exists(), + "failed publication leaked its staging file" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn failed_atomic_replace_preserves_destination_and_cleans_temp() { + let root = scratch_dir("atomic-replace-failure"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("sdk.tar.gz"); + fs::write(&destination, b"published").unwrap(); + + write_file_atomically_with_publish( + &destination, + b"replacement", + |attempt| OsString::from(format!("replace-failure-{attempt}")), + || {}, + |tmp, path| { + assert_eq!(fs::read(tmp).unwrap(), b"replacement"); + assert_eq!(path, destination); + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "simulated atomic replacement failure", + )) + }, + ) + .expect_err("simulated replacement failure must be returned"); + + assert_eq!(fs::read(&destination).unwrap(), b"published"); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); + } + + #[cfg(unix)] + #[test] + fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let file_name = OsString::from_vec(b"sdk-\xff.tar.gz".to_vec()); + let destination = Path::new("/tmp").join(&file_name); + let temp = temp_sibling_path(&destination, OsStr::new("collision")).unwrap(); + + let mut expected = file_name.into_vec(); + expected.extend_from_slice(b".tmp-collision"); + assert_eq!(temp.file_name().unwrap().as_bytes(), expected); + } + + /// Regression: a failed write must not leave a `.tmp-*` scratch file + /// behind. The name is unique per attempt, so before this an orphan + /// accumulated per retry — and when the failure is a full disk, those + /// orphans are exactly what keeps it full. + /// + /// Provokes the failure by pointing the destination at a non-empty + /// directory: the temp file is written, then neither the rename nor the + /// replace fallback can succeed. Portable, unlike an out-of-space test. + #[test] + fn write_file_atomically_cleans_up_temp_when_the_rename_fails() { + let root = scratch_dir("atomic"); + let occupied = root.join("sdk.tar.gz"); + fs::create_dir_all(occupied.join("nested")).unwrap(); + fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); + + write_file_atomically(&occupied, b"payload") + .expect_err("renaming onto a non-empty directory should fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| name.contains(".tmp-")) + .collect(); + let _ = fs::remove_dir_all(&root); + assert!( + leftovers.is_empty(), + "failed write left temp files behind: {leftovers:?}" + ); + } + + /// Mirrors the `/dev/shm` reproduction from the original report: a genuine + /// ENOSPC, not a rename failure standing in for one. + /// + /// Ignored by default because it fills `/dev/shm`, which is shared with + /// anything else on the host, so it is not safe to run concurrently. Run + /// with `cargo test -p rocm-core -- --ignored write_file_atomically_cleans_up`. + #[test] + #[ignore = "fills /dev/shm to provoke ENOSPC; not safe to run concurrently"] + fn write_file_atomically_cleans_up_temp_on_write_failure() { + let shm = Path::new("/dev/shm"); + if !shm.is_dir() { + eprintln!("skipping: /dev/shm unavailable"); + return; + } + let dir = shm.join(format!("rocm-core-enospc-{}", std::process::id())); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("artifact.tar.gz"); + // Larger than the tmpfs, so the write is guaranteed to hit ENOSPC. + let payload = vec![0u8; 256 * 1024 * 1024]; + + let mut failures = Vec::new(); + for _ in 0..2 { + write_file_atomically(&dest, &payload) + .expect_err("writing past the end of the filesystem should fail"); + failures.push( + fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(), + ); + } + let destination_exists = dest.exists(); + let _ = fs::remove_dir_all(&dir); + + for leftovers in &failures { + assert!( + leftovers.is_empty(), + "failed write left files behind: {leftovers:?}" + ); + } + assert!( + !destination_exists, + "destination must not exist after failure" + ); + } +} diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 58e59acfe4..62eaf05fad 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -1151,7 +1151,7 @@ last_installed_runtime_id = "therock-release" /// A directory at the destination is what makes the rename fail on both /// supported platforms without needing permissions a root CI lane ignores — /// the same device `write_file_atomically_cleans_up_temp_when_the_rename_fails` - /// uses in `apps/rocm`. + /// uses in `rocm_core::atomic_write`. #[test] fn a_config_save_that_cannot_publish_cleans_up_its_temp_file() -> Result<()> { let (root, paths) = temp_app_paths("config-save-rename-fails"); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aad2dda627..c8075f47a2 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -10,6 +10,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; pub mod amd_smi; #[cfg(all(test, unix))] mod archive_props; +pub mod atomic_write; pub mod automation; pub mod binary_paths; pub mod browser; From b680a8066a32ad15d5a60c2d13443176d66c0e47 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 12:29:07 +0000 Subject: [PATCH 02/17] test(runtimes): pin the active-runtime marker's atomic replacement `write_active_runtime_marker` used to stage `runtimes/active.json` at `active.json.tmp-` with a truncating write, remove the marker, then rename the temp file into place (#605). Two activations against one data dir in the same millisecond therefore wrote through one shared temp file. One of them failed with "failed to move active runtime marker". The other could publish its payload followed by the tail of a longer one, which later readers such as `rocm runtimes uninstall` could not parse. Between the remove and the rename, the marker did not exist at all. write itself does not change here beyond going through `rocm_core::atomic_write::write_file_atomically` (the one writer the previous commit moved into rocm-core) for both the forward write and the failed-activation restore. Each writer stages into its own `create_new` temp file named with pid, millis and attempt, and the marker is replaced in one step rather than removed first. The doc comment now says what that does and does not guarantee for two concurrent activations. Two regression tests pin it: - a deterministic one pre-creates a file at every name the old scheme could pick during the test and asserts none is written through or moved; - a Unix-only stress test runs two writers in lock-step plus a reader, and asserts every write succeeds and the reader always finds a complete marker. Against the pre-#424 body, 190 of 400 writes failed. Re-adding only the remove before the publish made the reader miss the marker thousands of times. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 209 ++++++++++++++++++++++++++++++++-- 1 file changed, 198 insertions(+), 11 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 29a7693588..b3360539b6 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -843,12 +843,16 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { } } +/// Replace the active-runtime marker in one step. +/// +/// Two `rocm` processes can activate a runtime against the same data +/// directory at once (two terminals, a script), so the write goes through the +/// shared atomic writer: each writer stages into its own exclusively created +/// temp file, and the marker is replaced rather than removed first, so a +/// reader always finds a complete marker. It does not order the two +/// activations: the last one to publish wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); - fs::create_dir_all( - path.parent() - .context("active runtime marker path has no parent directory")?, - )?; // The shared helper rather than a local temp-file-and-rename: it is the same // write the failed-activation restore uses, and it is strictly stronger than // a bare `fs::rename`, which on Windows is `MoveFileExW` and fails with a @@ -858,13 +862,12 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> // window with no marker at all, which runtime resolution and the storage // retention holds both read as "nothing is active" — and a failed publish // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. - rocm_core::atomic_write::write_file_atomically( - &path, - &serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?, - ) - .with_context(|| format!("failed to write {}", path.display()))?; - Ok(()) + // the folder `rocm runtimes` reads. It also creates the `runtimes` folder + // when it is missing. + let bytes = + serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; + rocm_core::atomic_write::write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write active runtime marker {}", path.display())) } #[cfg(test)] @@ -1194,6 +1197,190 @@ pub(crate) mod tests { Ok(()) } + fn test_active_runtime_marker(paths: &AppPaths, runtime_key: &str) -> ActiveRuntimeMarker { + ActiveRuntimeMarker { + runtime_id: "therock-release:gfx120X-all".to_owned(), + runtime_key: runtime_key.to_owned(), + manifest_path: runtime_manifest_path(paths, runtime_key), + install_root: paths.data_dir.join("runtimes").join(runtime_key), + previous_runtime_id: None, + previous_runtime_key: None, + activated_at_unix_ms: rocm_core::unix_time_millis(), + } + } + + fn leftover_marker_temp_files(paths: &AppPaths) -> Result> { + let dir = active_runtime_marker_path(paths) + .parent() + .context("marker has a parent")? + .to_path_buf(); + let mut leftovers = Vec::new(); + for entry in fs::read_dir(dir)? { + let name = entry?.file_name().to_string_lossy().into_owned(); + if name.starts_with("active.json.tmp-") { + leftovers.push(name); + } + } + Ok(leftovers) + } + + /// Regression for #605: the marker used to be staged at + /// `active.json.tmp-` with a truncating write, so a writer that + /// drew the same millisecond as another activation wrote into *that* + /// writer's temp file and then moved it into place as the marker. + /// + /// Pre-creating a file at every name that scheme could pick while this + /// test runs stands in for the other writer deterministically: the write + /// has to succeed without touching any of them. + #[test] + fn active_runtime_marker_write_never_reuses_another_writers_temp_file() -> Result<()> { + const WINDOW_MS: u128 = 2_000; + const FOREIGN: &[u8] = b"another writer's staged marker"; + let (root, paths) = test_paths("marker-temp-collision"); + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "previous"))?; + let marker_path = active_runtime_marker_path(&paths); + + let start = rocm_core::unix_time_millis(); + let foreign: Vec = (start..start + WINDOW_MS) + .map(|millis| marker_path.with_extension(format!("json.tmp-{millis}"))) + .collect(); + for path in &foreign { + fs::write(path, FOREIGN)?; + } + + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "next"))?; + let finished = rocm_core::unix_time_millis(); + assert!( + finished < start + WINDOW_MS, + "inconclusive: the write ran after the {WINDOW_MS} ms window of pre-created names" + ); + + let published: ActiveRuntimeMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; + assert_eq!(published.runtime_key, "next"); + for path in &foreign { + assert_eq!( + fs::read(path).ok().as_deref(), + Some(FOREIGN), + "{} belongs to another writer but was written through or moved", + path.display() + ); + } + for path in &foreign { + fs::remove_file(path)?; + } + assert_eq!(leftover_marker_temp_files(&paths)?, Vec::::new()); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// Regression for #605: two activations writing the marker at once must + /// both succeed, and a reader must find a complete marker throughout — + /// the old writer removed `active.json` before renaming the replacement + /// in, and two writers in the same millisecond shared one temp file, so + /// one failed with "failed to move active runtime marker" and the other + /// could publish the shorter payload followed by the longer one's tail. + /// + /// The bug is a race, so against the old code this fails with high + /// probability per round rather than deterministically; against the fix + /// nothing in it is timing-dependent. Unix only: the property under test + /// is that `rename` replaces in one step. The Windows publish path goes + /// through `ReplaceFileW` instead, whose behaviour with two replacements + /// of one file in flight this test has not been run against. + #[cfg(unix)] + #[test] + fn concurrent_active_runtime_marker_writes_keep_a_complete_marker_published() -> Result<()> { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Barrier}; + + const ROUNDS: usize = 200; + let (root, paths) = test_paths("marker-concurrent"); + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "initial"))?; + let marker_path = active_runtime_marker_path(&paths); + + let done = Arc::new(AtomicBool::new(false)); + let reader = { + let done = Arc::clone(&done); + let marker_path = marker_path.clone(); + std::thread::spawn(move || { + let mut problems = Vec::new(); + while !done.load(Ordering::Acquire) { + match fs::read(&marker_path) { + Ok(bytes) => { + if let Err(error) = + serde_json::from_slice::(&bytes) + { + problems.push(format!("unparsable marker: {error}")); + } + } + Err(error) => problems.push(format!("marker unreadable: {error}")), + } + } + problems + }) + }; + + let barrier = Arc::new(Barrier::new(2)); + let writers: Vec<_> = [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload", + ] + .into_iter() + .map(|runtime_key| { + let barrier = Arc::clone(&barrier); + let paths = paths.clone(); + std::thread::spawn(move || { + let mut errors = Vec::new(); + for _ in 0..ROUNDS { + barrier.wait(); + if let Err(error) = write_active_runtime_marker( + &paths, + test_active_runtime_marker(&paths, runtime_key), + ) { + errors.push(format!("{error:#}")); + } + } + errors + }) + }) + .collect(); + + let mut writer_errors = Vec::new(); + for writer in writers { + writer_errors.extend(writer.join().expect("writer thread panicked")); + } + done.store(true, Ordering::Release); + let reader_problems = reader.join().expect("reader thread panicked"); + + let published: ActiveRuntimeMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; + let leftovers = leftover_marker_temp_files(&paths)?; + let _ = fs::remove_dir_all(root); + + assert!( + writer_errors.is_empty(), + "{} of {} marker writes failed; first: {:?}", + writer_errors.len(), + 2 * ROUNDS, + writer_errors.first() + ); + assert!( + reader_problems.is_empty(), + "a reader found no complete marker {} times; first: {:?}", + reader_problems.len(), + reader_problems.first() + ); + assert!( + [ + "short", + "a-runtime-key-long-enough-to-leave-a-tail-behind-a-shorter-payload" + ] + .contains(&published.runtime_key.as_str()), + "unexpected marker: {published:?}" + ); + assert_eq!(leftovers, Vec::::new()); + Ok(()) + } + #[test] fn runtime_activation_records_exact_key_and_rollback() -> Result<()> { let (root, paths) = test_paths("runtime-activation"); From c83ef2326726e49992a8e676a1443073911462c0 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 12:29:07 +0000 Subject: [PATCH 03/17] docs(architecture): point at rocm-core's shared atomic writer The rocmd sandbox module no longer carries its own atomic-write helpers; record where the shared one lives so the next caller reuses it. Signed-off-by: Roman Inflianskas --- docs/architecture.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index ad51f2aaf6..0b5da7b030 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -103,9 +103,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and reads the os-release every one of these plans is built from through `rocm_core::hardware_root::real_os_release`, which has no copy here. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs). `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `rocm_core::hardware_root::real_os_release` and the three OpenMPI/libatomic/libnuma install hints ignore a simulated host root, and a scan that no source file of the binary reads os-release by hand. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install orchestration and is the parent of the `therock/` children below. It keeps `install_sdk` and `install_sdk_for_update` (with `SdkInstallRequest` and `SdkInstallResult`), the wheel and tarball install paths (`install_wheel_runtime`, `install_tarball_runtime`) with the provenance and notes they print (`render_canonical_provenance`, the host-newer-ROCm and no-wheels messages, `quote_display_arg`), and the helpers that name an install and its folders: `runtime_key`, `wheel_runtime_key`, `managed_runtime_root`, `resolved_install_root`, `venv_python_path`, `platform_tarball_token` and `slugify`, plus `ensure_tarball_install_target_is_clean`, the refusal to extract a tarball into a root that already holds a payload. What "holds a payload" means is not defined here: it is `rocm_core::install_root_has_payload`, the one rule shared with `runtime_registry.rs` and `therock/update.rs`. The runtime-manifest model and registry loaders it writes through live in [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs); `therock.rs` re-exports the names other modules reach as `therock::...`: `InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `load_runtime_manifests` and `validate_rocm_sdk_runtime_probe` from `runtime_registry.rs` (plus `RocmSdkLibraryProbe`, for tests only), and from its child modules the `pub(crate)` names each child's entry below lists. The children read five of them from here, so they stay in `therock.rs` for good: `slugify` ([`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) and [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs)), `venv_python_path` ([`therock/python.rs`](../apps/rocm/src/therock/python.rs)), `platform_tarball_token` ([`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs)), and `runtime_key` and `wheel_runtime_key` ([`therock/update.rs`](../apps/rocm/src/therock/update.rs)). The `runtimes` subcommand's command enum (`RuntimesCommand`) lives in [`cli.rs`](../apps/rocm/src/cli.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). - [`therock/version.rs`](../apps/rocm/src/therock/version.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): the version grammar behind runtime selection and update checks. `parse_version` is the strict gate an install is routed on; `parse_version_for_ordering`, `compare_version_strings` and `version_relation` implement the PEP 440 ordering and the "is this a different version" question; `parse_host_version` reads a host-reported ROCm version through the same parser; `runtime_version_display`, `runtime_version_build_date`, `normalize_requested_build_date` and the calendar helpers behind them handle build dates; `is_stable_runtime_version` and the `ParsedVersion`/`VersionOrderKey`/`VersionStage` types back them. The items `therock.rs` and its property-test module use are `pub(super)` (including the `ParsedVersion` and `VersionOrderKey` fields they read); the parsing and calendar helpers behind them are private; and `runtime_version_display` is `pub(crate)`, re-exported by `therock.rs` for the other modules that render versions. Depends on nothing else in `therock`. `therock_version_proptests.rs` stays declared from `therock.rs` because it also reaches update, wheel and registry code. -- [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`) and the stage-then-publish atomic file writes (`write_file_atomically`, `stage_file_for_atomic_publish`, `temp_sibling_path`, and the platform `publish_temp_file`/`replace_file_windows` pair) for the metadata cache entries and the staged metadata public key, the startup update-check record and the active-runtime marker; runtime registry manifests do not go through them. `write_file_atomically` is `pub(crate)` and re-exported by `therock.rs` for [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which writes the active-runtime marker with it; the rest is `pub(super)` or private. The `cfg(windows)` publish and replace paths compile only on the Windows lane. +- [`therock/io.rs`](../apps/rocm/src/therock/io.rs) — full domain extraction, a leaf child of `therock.rs` (split out of it): unique scratch directories (`linux_temp_dir`, `windows_temp_dir`), both `pub(super)`. The atomic file writes that used to live here are rocm-core's shared [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs), which [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs), [`therock/update.rs`](../apps/rocm/src/therock/update.rs) and [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) call directly. - [`therock/process.rs`](../apps/rocm/src/therock/process.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for scratch directories (split out of `therock.rs`): running and capturing child commands (`run_command*`, `capture_command_output*`, `capture_python_stdout*`, `run_uv_progress_command`, and the Windows temp-file variants with `windows_child_path`), the bounded wait (`wait_with_output_bounded`), the progress-line output with its `--json` suppression guard (`SuppressProgressOutput`, `progress_line`), and unpacking the SDK tarball (`extract_tarball`, `extract_tarball_and_discard_archive`, which is why it sits here and not in `io.rs`: it spawns `tar` through `run_command`). Its `archive_properties` property tests run on Unix only. The process-wide environment lock these tests take is the single `PROCESS_ENV_TEST_LOCK` in [`test_support.rs`](../apps/rocm/src/test_support.rs), shared with the tests left in `therock.rs`. -- [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) — full domain extraction, a child of `therock.rs` that depends on `therock/io.rs` for atomic writes and on `slugify` in the parent for cache file names (split out of `therock.rs`): HTTP fetching for installs. `download_text_cached` is the metadata GET with its on-disk etag cache (`CachedHttp*`) and detached-signature verification (`MetadataSignaturePolicy`, `PINNED_METADATA_PUBLIC_KEY_PEM`, the unset trust root that `cargo xtask verify-pinned-keys` checks against the canonical published key, so its path is pinned in `xtask`); `download_file` is the streaming artifact download; `head_content_length` and `preflight_tarball_space` size the download and check free space first; `http_get` is the shared request. `truthy_env` is `pub(crate)` and re-exported by `therock.rs` for [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs); `download_text_cached`, `download_file`, `preflight_tarball_space` and the `CachedHttpText` struct (with its `text` field) that `download_text_cached` returns are `pub(super)` for the install and resolution code that stays in `therock.rs`; everything else is private. +- [`therock/fetch.rs`](../apps/rocm/src/therock/fetch.rs) — full domain extraction, a child of `therock.rs` that depends on rocm-core's [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) for atomic writes and on `slugify` in the parent for cache file names (split out of `therock.rs`): HTTP fetching for installs. `download_text_cached` is the metadata GET with its on-disk etag cache (`CachedHttp*`) and detached-signature verification (`MetadataSignaturePolicy`, `PINNED_METADATA_PUBLIC_KEY_PEM`, the unset trust root that `cargo xtask verify-pinned-keys` checks against the canonical published key, so its path is pinned in `xtask`); `download_file` is the streaming artifact download; `head_content_length` and `preflight_tarball_space` size the download and check free space first; `http_get` is the shared request. `truthy_env` is `pub(crate)` and re-exported by `therock.rs` for [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs); `download_text_cached`, `download_file`, `preflight_tarball_space` and the `CachedHttpText` struct (with its `text` field) that `download_text_cached` returns are `pub(super)` for the install and resolution code that stays in `therock.rs`; everything else is private. - [`therock/source.rs`](../apps/rocm/src/therock/source.rs) — full domain extraction, a child of `therock.rs` that depends only on `therock/version.rs` (split out of `therock.rs`): where an install reads from. `TheRockChannel` (release or nightly) and `SourceLayout` (the canonical stream, or ROCm 10's separately hosted next layout) are the two axes; `resolve_source` maps a pair to the `ResolvedSource` wheel-index and tarball-catalog bases, honouring a `ROCM_CLI_THEROCK_*_BASE` override only behind the explicit opt-in (`therock_base_override_allowed`, `select_base_override`, `env_override_base`); `select_source_layout`, `requested_source_layout` and `next_layout_requested` decide the layout from a pin; `RuntimeVersionSelector` is the version or build date a user pins (`pub(crate)`, re-exported by `therock.rs`). `FamilyResolution` lives here rather than in `therock/family.rs` because choosing the layout reads it; that keeps the dependency one-way, with `family.rs` building on this file. `TheRockChannel`, `SourceLayout`, `ResolvedSource` and `FamilyResolution` (with their methods and fields), `RuntimeVersionSelector`'s `describe` and `matches_version`, `resolve_source`, the three layout functions, and the `THEROCK_SOURCE_LAYOUT_GENERATION` and `THEROCK_NEXT_MIN_MAJOR` constants are `pub(super)` for `therock.rs` and its siblings; the base-override helpers and the other constants (the per-channel and next-layout base URLs, and the next layout's generation tag) are private to this file. - [`therock/family.rs`](../apps/rocm/src/therock/family.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` for `TheRockChannel` and `FamilyResolution` (split out of `therock.rs`): which package family a request targets. `resolve_family` picks it from `--family`, `ROCM_CLI_THEROCK_FAMILY`, a managed runtime or the host GPU, in that order, and refuses a GPU with no ROCm build (`unsupported_host_gpu_message`, `unsupported_requested_target_message`) instead of substituting another family; `family_override_or_recovered_arch` and `raw_arch_agreeing_with_family` recover the exact arch behind a grouped family; `family_resolution_hint` is the recovery text appended when no runtime is found. `resolve_family`, `family_override_or_recovered_arch` and `family_resolution_hint` are `pub(super)`; the rest is private. - [`therock/python.rs`](../apps/rocm/src/therock/python.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/io.rs`, `therock/process.rs` and `therock/source.rs` (split out of `therock.rs`): the Python an install runs on. `python_requirement` maps a `SourceLayout` to the interpreter the layout needs (`PythonRequirement`: wheel tag and `uv` version), which is why `source.rs` had to move first; `resolve_python_launcher` finds one in order of an explicit `ROCM_CLI_PYTHON`, `PATH`, the saved managed Python, then `ensure_managed_python`, which installs one through `uv` and records it in the managed-Python manifest; the resolver reads its environment through `PythonResolverEnv`, so tests steer it with a thread-local override (`OverridePythonResolverEnv`) rather than rewriting `PATH`; `wheel_compatibility_for_python` and `current_platform_wheel_tags` report the wheel tags an interpreter and platform accept (`WheelCompatibility`); `ensure_uv_venv` creates, or recreates when the interpreter tag no longer matches, the runtime's virtual environment. `venv_python_path` stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The items `therock.rs` reaches, and `split_runtime_path`, which `therock/probe.rs` reads, are `pub(super)`; the rest is private, including `ensure_managed_python` and `resolve_python_launcher_in` (the resolver's entry point that takes its environment as an argument). Its Unix-only tests drive fake interpreters and `uv` scripts. @@ -114,7 +114,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`therock/resolve/tarball.rs`](../apps/rocm/src/therock/resolve/tarball.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/fetch.rs`, `therock/family.rs`, `therock/source.rs` and `therock/version.rs` (split out of `therock.rs`): resolving the SDK tarball a request installs. `resolve_tarball_artifact` (and `resolve_tarball_artifact_with_timeout`, which takes the metadata download timeout) resolves the family and source layout, downloads the catalog through the cached metadata fetch, and returns the `TarballArtifact` it chose; `parse_tarball_index_html` reads the catalog's embedded file list into `TarballIndexFile`s and `validate_tarball_file_name` refuses an entry that is not a plain file name; `select_tarball_candidate` picks the newest archive the request can install, with `tarball_family_token` spelling the family the way the next layout's catalog does. `platform_tarball_token`, which names the archives' platform, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. Only `resolve_tarball_artifact*` and `TarballArtifact` are visible beyond this file. - [`therock/resolve/wheel.rs`](../apps/rocm/src/therock/resolve/wheel.rs) — full domain extraction, a child of `therock/resolve.rs` that builds on `therock/family.rs`, `therock/fetch.rs`, `therock/process.rs`, `therock/python.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for `WheelRuntimeComposition` (split out of `therock.rs`): resolving the wheel stack a request installs. `resolve_pip_runtime` (and `resolve_pip_runtime_with_timeout`, which takes the metadata download timeout and an optional source layout) resolves the family and the layout, reads the aggregate root, decides the device payload and returns a `PipRuntimeResolution`; on the canonical layout `select_matching_pip_package_versions` then takes the newest `rocm` whose `torch`, `torchvision` and `torchaudio` listings agree, and on the next layout `torchaudio_pin` pins a torchaudio first (the hardware-verified release when it is published, read from the vLLM engine's table) and `resolve_published_pip_package_versions` has `uv pip compile` choose the rest. `wheel_runtime_composition` turns a resolution into the exact package specs (`therock_pip_package_specs`) an install gives `uv` and records in the manifest, `published_pip_requirements` is the matching requirement set for `uv pip compile`, and `canonical_auto_select_needs_next_python` peeks at the channel's own `rocm` listing so the interpreter can be picked before resolution. Beneath that, `parse_aggregate_package_links` and `parse_aggregate_device_targets` read the package links of the canonical aggregate root and the exact device targets it publishes, and `validate_aggregate_index_layout` refuses a root that lacks the expected packages or any device payload; `AggregateDeviceTarget::resolve` picks the one exact target a host gets, or records why it could not (`Undetermined`); `load_simple_index_versions` fetches a package's simple-index listing through the cached metadata download and `parse_simple_index_versions` reads the versions out of it, dropping wheels the interpreter's tags do not accept when given them (`wheel_stem_matches_compatibility` and the `wheel_*_tag_matches` helpers); `package_release_key` orders the PEP 440 versions found there (`PackageReleaseKey`). `slugify`, which keys the cached listing, stays in `therock.rs` for good, with the other keys and roots helpers, and this module reads it from there. The functions and types `therock.rs` and `therock/update.rs` name are `pub(in crate::therock)` and reach them through `therock/resolve.rs`; `PipRuntimeResolution` and `TheRockPipPackageVersions` are `pub(in crate::therock)` with the fields they read, and `TorchaudioPin` with its `describe`, so they can use what `resolve_pip_runtime` returns without naming the types; the rest is private. - [`therock/consent.rs`](../apps/rocm/src/therock/consent.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/source.rs` and `therock/version.rs` and reaches the registry through [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) (split out of `therock.rs`): the consent gate for an install that would displace the active default runtime. `active_default_runtime_relation` says what the install would replace (an upgrade, downgrade or reinstall when it is the same family and channel, otherwise the different runtime it displaces) or `None` when nothing is the active default, and it fails closed, with a relation naming what could not be resolved, when the config names an active default the registry cannot resolve; `SdkInstallConsent` and `SdkInstallApprovalSource` say how consent is obtained (asked for, or already granted by `--yes`, `--approve-replacing-active-default` or `rocm update --apply`); `sdk_install_approval` turns those and whether a terminal is attached into proceed, prompt or refuse (`SdkInstallApproval`); `preapproved_install_line`, `fresh_install_line`, `refuse_non_interactive_message` and `confirm_overwrite_existing_sdk` are the wording of each outcome, and the prompt reads its answer through `prompt_yes_no` and the shared [`confirm.rs`](../crates/rocm-core/src/confirm.rs). `SdkInstallConsent`, `SdkInstallApprovalSource` and `preapproved_install_line` are re-exported by `therock.rs`; the other items `therock.rs` calls are `pub(super)`, as is `active_default_relation_text`, which the version property tests call, and the rest is private. -- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/io.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, and on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. +- [`therock/update.rs`](../apps/rocm/src/therock/update.rs) — full domain extraction, a child of `therock.rs` that builds on `therock/family.rs`, `therock/process.rs`, `therock/python.rs`, `therock/resolve.rs`, `therock/source.rs` and `therock/version.rs`, on [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) for the manifests and on rocm-core's [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) for the startup-check record (split out of `therock.rs`): whether an installed runtime has something newer, and the bounded startup check that asks. `runtime_update_plan` resolves the latest source for one runtime (`resolve_latest_for_manifest` re-resolves a wheel runtime through `resolve_pip_runtime_with_timeout` and a tarball one through `resolve_tarball_artifact_with_timeout`, on the layout `manifest_source_layout` reads back from the manifest) and `runtime_freshness` classifies it (`RuntimeFreshness`: up to date, update available, repair available when the installed composition is not the one a fresh install would record, or ahead of the index) into a `RuntimeUpdatePlan`. `render_update_report` and `render_update_json` (the form the dashboard's update check parses) present the plans, and `maybe_refresh_startup_update_check` keeps one `StartupUpdateCheckRecord` in the cache so the check runs at most once every twelve hours, under a two-second download timeout. `runtime_key` and `wheel_runtime_key`, which name the install a plan targets, stay in `therock.rs`; whether a replacement's root is installed is `rocm_core::install_root_has_payload`. `render_update_report`, `render_update_json`, `runtime_update_plan` and `maybe_refresh_startup_update_check` are `pub(crate)` and re-exported by `therock.rs`; `RuntimeFreshness`, `runtime_freshness` and `select_startup_update_manifest` are `pub(super)` for the version property tests, and the rest is private. - [`therock/test_support.rs`](../apps/rocm/src/therock/test_support.rs) — test-only (`#[cfg(test)]`), a child of `therock.rs`: fixtures shared by the `therock` child modules' tests and the tests left in `therock.rs` (`test_paths`, `test_runtime_manifest`, `write_test_runtime_manifest`, `family_for_test`). Separate from the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs), which holds the fixtures every module's tests share, such as `PROCESS_ENV_TEST_LOCK` and the background-helper fixtures `runtime_state` and `set_helper_facts`. - [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) — the chat assistant's tool-call policy: which `rocm` commands and bridge tools a model may request, which run immediately and which need the user's approval first. It owns `ChatRocmCommandAction` (read-only, or approval with the titles the approval prompt shows) and `ChatToolApprovalRequest`, the validators (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `validate_required_chat_string`, `validate_optional_chat_integer`, `reject_control_characters_in_tool_arguments` (which refuses a model-supplied tool name or argument carrying a terminal control character, for every tool except `rocm_command`, whose argv `normalized_chat_rocm_command_args` screens entry by entry), `validate_serve_command_safety`, the four rules a model-proposed `serve` must satisfy (no public bind, no CPU device, a loopback host, `--managed` without `--foreground`), and `validate_chat_rocm_command_safety`, which applies them to a `serve` argv and also refuses a system install folder (`chat_install_prefix_is_system`)), the classifiers (`chat_rocm_command_action`, `chat_rocm_command_action_from_args`, `chat_install_driver_action`, `canonicalize_chat_rocm_command`, `normalized_chat_rocm_command_args`, `chat_tool_call_is_read_only`), `chat_tool_approval_request`, and the readers that turn a tool call into its argv and one-line command (`rocm_chat_tool_requested_args`, `rocm_chat_tool_requested_command`, with `push_optional_json_cli_arg` and `json_string`). It moved verbatim from `chat.rs` with the tests that exercise only these items. It depends on [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `providers::ChatToolCall`, `service_launch::is_loopback_host`, `services_text::validate_service_id`, `shell_quote::has_terminal_control_character`, `therock::RuntimeVersionSelector` and `driver_install::host_install_driver_only_reports`, none of which imports `chat`, so `freeform_planner.rs` calls `validate_serve_command_safety` and `validate_chat_rocm_command_safety` here instead of through `chat`, and keeps no copy of the serve rules. `chat.rs`'s child modules call into it and `chat.rs` re-exports `validate_chat_tool_call`, `chat_tool_call_is_read_only` and `chat_tool_approval_request` (read by `dash_seam.rs`), and `ChatRocmCommandAction`, `chat_install_driver_action`, `chat_rocm_command_action_from_args` and `rocm_chat_tool_requested_args`, which only other modules' tests read as `crate::chat::` (`driver_plan.rs`, `system_packages.rs`). - [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. The two modules depend on each other: `uninstall_plan.rs` reads this module's `pub(crate)` `PathEdit`, `PATH_EDIT_RECORD`, `plan_path_edits`, `plan_is_empty`, `removal_count` and `render_extra_plan`. @@ -150,7 +150,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). - [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. -- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. +- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. @@ -171,6 +171,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. +- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the one crash- and race-safe file replacement both `rocm` and `rocmd` use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); add a caller rather than another copy. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. @@ -216,7 +217,7 @@ What else stays in [`lib.rs`](../crates/rocm-core/src/lib.rs), beside module dec - `sanitize_component` — the lowercase-alphanumeric slugger shared by [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs)'s `generate_service_id` and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs)'s artifact cache-marker names. It is not model-catalog-specific, so it does not move there. - `TORCH_ALIGNMENT_DISABLED_ENV`/`torch_alignment_disabled` — the one read of the torch-alignment opt-out, shared by the CLI and the vLLM engine so the two cannot disagree. - `VLLM_GPU_MEMORY_UTILIZATION_HINT` — a single wording contract between the CLI, the vLLM engine, and this crate's own `fix-16-vllm-oom` diagnosis; `rocm-core` is the only crate all of them can reach. -- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The open [#610](https://github.com/ROCm/rocm-cli/pull/610) adds a shared, stronger atomic writer to this crate for `rocm` and `rocmd`, and the open [#541](https://github.com/ROCm/rocm-cli/pull/541) moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). Once #610 lands, the automation and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. +- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The shared, stronger writer `rocm` and `rocmd` use is [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs); [#541](https://github.com/ROCm/rocm-cli/pull/541), which builds on it, moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). The automation, JSONL-store and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. Once the `write_atomic` move lands, [`lib.rs`](../crates/rocm-core/src/lib.rs) holds only module declarations, re-exports, the crate-root primitives above, the private `use` shims that let sibling modules reach some `host_gpu`, `managed_runtime`, `rocm_install` and `runtime` items as `crate::` paths, and its test-only modules — the end state of the modularization effort for this crate (ROCMAI-27, planned last: highest fan-in, since every app and engine crate depends on it, but lowest novelty, since the sibling modules already prove the pattern). From 463bc5c0974ac4cf88b1fabdbd54cdd2cac2f947 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 12:46:50 +0000 Subject: [PATCH 04/17] style(core): give the atomic writer's test seams short doc summaries clippy's too_long_first_doc_paragraph rejects the two seam doc comments carried over from therock, where the private functions were not linted for it. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/atomic_write.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index 5a19aca30b..631294a9ae 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -60,8 +60,10 @@ pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { ) } -/// [`write_file_atomically`] with the temp-name suffix for each attempt and a -/// hook that runs after the temp file is fully written and before it is +/// [`write_file_atomically`] with test seams for the temp name and publish point. +/// +/// `suffix_for_attempt` names the temp file for each attempt, and +/// `before_publish` runs after the temp file is fully written and before it is /// published. Both exist so tests can force a temp-name collision and hold /// writers at the publish point; production code calls /// [`write_file_atomically`]. @@ -84,10 +86,11 @@ where ) } -/// [`write_file_atomically_with`] with the publish step injected as well, so -/// tests can observe the staged file and the destination at the moment of -/// publication, or make the publication fail. Pass [`publish_temp_file`] for -/// the real one. +/// [`write_file_atomically_with`] with the publish step injected as well. +/// +/// Tests use it to observe the staged file and the destination at the moment +/// of publication, or to make the publication fail. Pass +/// [`publish_temp_file`] for the real one. pub fn write_file_atomically_with_publish( path: &Path, bytes: &[u8], From 0ef5fcdba090ccf22b53654210c200d17915e5fa Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 12:56:10 +0000 Subject: [PATCH 05/17] fix(rocmd): gate std::fs import to the targets that still use it With the atomic writer moved to rocm-core, sandbox.rs uses std::fs only in its Linux bubblewrap path and its tests, so the Windows build warned about an unused import. Signed-off-by: Roman Inflianskas --- apps/rocmd/src/sandbox.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 4bb1902939..9eaf6c8c4a 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -18,6 +18,7 @@ use serde_json::Value; use serde_json::json; #[cfg(test)] use sha2::Digest; +#[cfg(any(test, target_os = "linux"))] use std::fs; use std::path::{Path, PathBuf}; #[cfg(target_os = "linux")] From b52216b04d3655c192f8d7e7c2594e46de829da3 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 8 Oct 2026 13:06:43 +0000 Subject: [PATCH 06/17] ci(coverage): lower rocmd's floor for the tests that moved to rocm-core Moving the atomic writer out of rocmd's sandbox.rs took its helper code and six tests with it. Those lines were close to fully covered, and the line count includes test modules, so rocmd's percentage drops although nothing it still owns lost coverage. The code and its tests are now measured under rocm-core. Only rocm's tests reach rocmd, so `cargo llvm-cov --no-cfg-coverage -p rocm -p rocmd` measures rocmd exactly as the workspace run does. Measured locally that way, rocmd went from 75.30% (7776 lines) on the base to 75.02% (7549 lines) here. That is a 0.28pp drop, past the 0.2pp tolerance. The floor is set to the branch measurement. Only this one line is changed. A full `--bless` from a local run would also rewrite every other crate's floor from a different host's numbers. rocm moved by -0.04pp, inside tolerance, and is left as is. Signed-off-by: Roman Inflianskas --- coverage-floors.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coverage-floors.toml b/coverage-floors.toml index aa2ece4edd..3cc2e5514e 100644 --- a/coverage-floors.toml +++ b/coverage-floors.toml @@ -23,5 +23,5 @@ "rocm-engine-lemonade" = 63.82 # 5554 lines "rocm-engine-protocol" = 98.37 # 491 lines "rocm-engine-vllm" = 66.76 # 3030 lines -"rocmd" = 75.36 # 7714 lines +"rocmd" = 75.02 # 7549 lines "xtask" = 76.18 # 5447 lines From ea48bf41e584ae6c4a695fdef8716b25c81819ae Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 9 Oct 2026 08:05:39 +0000 Subject: [PATCH 07/17] fix(rocmd): drop the windows-sys dependency the move left unused `ReplaceFileW` was rocmd's only use of `windows-sys`; it moved into `rocm_core::atomic_write` with the rest of the atomic writer, and rocm-core declares its own dependency. Also narrow the marker regression test's doc comment: it guards against a return to the millis-only temp names, while collision safety under the current names is covered by the concurrent tests. Signed-off-by: Roman Inflianskas --- Cargo.lock | 1 - apps/rocm/src/runtimes_cmd.rs | 7 ++++++- apps/rocmd/Cargo.toml | 3 --- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..4a22bade0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3937,7 +3937,6 @@ dependencies = [ "sha2", "tokio", "ureq", - "windows-sys 0.61.2", ] [[package]] diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index b3360539b6..27150c2d60 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -1231,7 +1231,12 @@ pub(crate) mod tests { /// /// Pre-creating a file at every name that scheme could pick while this /// test runs stands in for the other writer deterministically: the write - /// has to succeed without touching any of them. + /// has to succeed without touching any of them. This guards against a + /// return to that millis-only scheme specifically. The shared writer's own + /// names (`{name}.tmp-{pid}-{millis}-{attempt}`) cannot match these by + /// construction, so collision safety under the current scheme is covered + /// by `concurrent_active_runtime_marker_writes_keep_a_complete_marker_published` + /// and by `rocm_core::atomic_write`'s own concurrency tests. #[test] fn active_runtime_marker_write_never_reuses_another_writers_temp_file() -> Result<()> { const WINDOW_MS: u128 = 2_000; diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..945a47e903 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -24,6 +24,3 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } - -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } From 8f94aaa8de8679ee7c31e823bb455dbb844de0fa Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 13 Oct 2026 16:31:33 +0000 Subject: [PATCH 08/17] build(rocm): drop the windows-sys dependency the move left unused ReplaceFileW was apps/rocm's only use of windows-sys, and it moved into rocm_core::atomic_write with the rest of the writer; rocm-core declares its own dependency. Same leftover as the rocmd one removed earlier. Signed-off-by: Roman Inflianskas --- Cargo.lock | 1 - apps/rocm/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a22bade0e..2151dc3607 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3741,7 +3741,6 @@ dependencies = [ "unicode-width", "ureq", "windows-native-keyring-store", - "windows-sys 0.61.2", "zbus-secret-service-keyring-store", ] diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..7d53b8c7d4 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -65,7 +65,6 @@ libc.workspace = true [target.'cfg(windows)'.dependencies] self-replace = "1.5" windows-native-keyring-store = "1.1" -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1.0", features = ["keychain"] } From f9de09a45b90b0afeddcbae5e470241bb9b1b3cc Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 13 Oct 2026 16:31:34 +0000 Subject: [PATCH 09/17] fix(core): state what the atomic writer guarantees, and test the full disk The module doc now gives the real reason for ReplaceFileW (a sharing violation while another process holds the destination open; std rename does replace), says Windows concurrent publishes are untested, and says atomic but not durable, since nothing is fsynced. The test seams no external production code calls are narrowed: write_file_atomically_with is private, and the two that apps/rocm's cross-crate tests drive stay pub but #[doc(hidden)] and say they are seams. The staging doc no longer implies an outside caller can publish and clean up on its own. The full-disk test called disk_space::map_write_error directly, so it stayed green if staging stopped mapping errors. It now drives the real staging path with the byte write swapped for one that fails like a full disk, and checks the message and that no temp file is left. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/atomic_write.rs | 115 ++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 22 deletions(-) diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index 631294a9ae..fc60e3c3ac 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -//! Crash- and race-safe file replacement. +//! Atomic file replacement, shared by `rocm` and `rocmd`. //! //! [`write_file_atomically`] stages the bytes in a fresh sibling temp file and //! then publishes it over the destination in one step, so a reader sees either @@ -13,9 +13,20 @@ //! reserved with `create_new`, so two writers — threads of one process or //! separate processes — can never write through the same temp file, even when //! their pid and millisecond collide; a collision just moves on to the next -//! attempt. A failed write or publish removes its temp file. On Windows the -//! publish uses `ReplaceFileW` when the destination already exists, because -//! `rename` onto an existing file is not a replace there. +//! attempt. A failed write or publish removes its temp file. +//! +//! On Windows, when the destination already exists, the publish uses +//! `ReplaceFileW` rather than `rename`. `std::fs::rename` does replace an +//! existing file there, but as `MoveFileExW` it fails with a sharing violation +//! while another process holds the destination open, which is the case +//! `ReplaceFileW` is used for. Two publishes of one file in flight at once +//! through `ReplaceFileW` are not exercised by any test; the concurrency tests +//! below run on Unix only. +//! +//! Atomic, not durable: neither the temp file nor its directory is fsynced +//! before the publish. That survives a crashed process and a concurrent +//! writer, but after a power loss a filesystem that does not order data before +//! the rename can still show an empty or stale destination. //! //! This makes each file individually atomic. It does not serialize a //! read-modify-write across processes: two writers that each read, change and @@ -67,7 +78,7 @@ pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { /// published. Both exist so tests can force a temp-name collision and hold /// writers at the publish point; production code calls /// [`write_file_atomically`]. -pub fn write_file_atomically_with( +fn write_file_atomically_with( path: &Path, bytes: &[u8], suffix_for_attempt: S, @@ -86,11 +97,17 @@ where ) } -/// [`write_file_atomically_with`] with the publish step injected as well. +/// Test seam: [`write_file_atomically`] with the temp name, a hook before the +/// publish, and the publish step itself injected. /// -/// Tests use it to observe the staged file and the destination at the moment -/// of publication, or to make the publication fail. Pass -/// [`publish_temp_file`] for the real one. +/// Tests use it to force a temp-name collision, to hold writers at the publish +/// point, to observe the staged file and the destination at the moment of +/// publication, or to make the publication fail. Pass [`publish_temp_file`] +/// for the real publish. Production code calls [`write_file_atomically`]. +/// +/// `pub` only because `apps/rocm`'s metadata-cache tests drive it across the +/// crate boundary; hidden from the docs because it is not an API to build on. +#[doc(hidden)] pub fn write_file_atomically_with_publish( path: &Path, bytes: &[u8], @@ -109,8 +126,12 @@ where } /// Write `bytes` to a fresh, uniquely named sibling of `path` and return the -/// staged file's path without publishing it. The caller owns the staged file: -/// it either publishes it or removes it. +/// staged file's path without publishing it. +/// +/// The caller owns the staged file and must remove it once done with it; on a +/// staging failure nothing is left behind. To *replace* `path` with the bytes, +/// call [`write_file_atomically`] instead, which also removes the staged file +/// when the publish fails. pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result { let temp_id = process_temp_id(); stage_file_for_atomic_publish_with(path, bytes, |attempt| { @@ -119,12 +140,30 @@ pub fn stage_file_for_atomic_publish(path: &Path, bytes: &[u8]) -> Result( + path: &Path, + bytes: &[u8], + suffix_for_attempt: S, +) -> Result +where + S: FnMut(u32) -> OsString, +{ + stage_file_for_atomic_publish_with_writer(path, bytes, suffix_for_attempt, |file, bytes| { + file.write_all(bytes) + }) +} + +/// [`stage_file_for_atomic_publish_with`] with the write into the reserved +/// temp file injected, so a test can make it fail the way a full disk does +/// without filling one. +fn stage_file_for_atomic_publish_with_writer( path: &Path, bytes: &[u8], mut suffix_for_attempt: S, + write: W, ) -> Result where S: FnMut(u32) -> OsString, + W: FnOnce(&mut fs::File, &[u8]) -> io::Result<()>, { let parent = path.parent().context("file path has no parent directory")?; fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; @@ -156,7 +195,7 @@ where ); }; - if let Err(error) = file.write_all(bytes) { + if let Err(error) = write(&mut file, bytes) { drop(file); let _ = fs::remove_file(&tmp); return Err(disk_space::map_write_error(error, &tmp)); @@ -181,16 +220,26 @@ where .with_context(|| format!("failed to publish {}", path.display())) } -/// Move the staged file `tmp` over `path` in one step, replacing any existing -/// file without first removing it. +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. #[cfg(not(windows))] +#[doc(hidden)] pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { fs::rename(tmp, path) } -/// Move the staged file `tmp` over `path` in one step, replacing any existing -/// file without first removing it. +/// Test seam: move the staged file `tmp` over `path` in one step, replacing +/// any existing file without first removing it. +/// +/// The real publish step to pass to [`write_file_atomically_with_publish`]. +/// `pub` only for that cross-crate test use; it does not remove `tmp` on +/// failure, so production code calls [`write_file_atomically`] instead. #[cfg(windows)] +#[doc(hidden)] pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { if path.try_exists()? { return replace_file_windows(path, tmp); @@ -253,15 +302,37 @@ mod tests { )) } + /// A write that runs out of space while staging says so in words, and + /// leaves no temp file behind to keep the disk full. Drives the real + /// staging path — the one both [`write_file_atomically`] and + /// [`stage_file_for_atomic_publish`] go through — with only the byte write + /// into the reserved temp file swapped for one that fails like a full + /// disk, so the mapping is checked where it is applied rather than on its + /// own. #[test] - fn write_file_atomically_reports_a_full_disk_clearly() { - // Exercise the mapping the write path uses, without filling a disk. - let error = disk_space::map_write_error( - std::io::Error::from(std::io::ErrorKind::StorageFull), - Path::new("/cache/rocm.tar.gz.tmp"), - ); + fn staging_into_a_full_disk_reports_it_clearly_and_removes_the_temp_file() { + let root = scratch_dir("atomic-full-disk"); + let destination = root.join("rocm.tar.gz"); + + let error = stage_file_for_atomic_publish_with_writer( + &destination, + b"payload", + |attempt| OsString::from(format!("full-disk-{attempt}")), + |_, _| Err(io::Error::from(io::ErrorKind::StorageFull)), + ) + .expect_err("a write that hits a full disk must fail"); + + let leftovers: Vec = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let _ = fs::remove_dir_all(&root); let text = format!("{error:#}"); assert!(text.contains("ran out of disk space"), "{text}"); + assert!( + leftovers.is_empty(), + "the failed staging left files behind: {leftovers:?}" + ); } /// The temp name keeps every extension, so a cleanup sweep over a cache From 304c1fa67cb1382b474c3a89c9afc51370b7efbd Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 13 Oct 2026 16:31:35 +0000 Subject: [PATCH 10/17] test(runtimes): describe the marker tests as what they pin after #424 #424 already fixed the temp-name collision and the missing-marker window on main. The two tests now say they pin that the marker keeps those properties through the shared writer, not that they are the regression for #605. The collision test no longer fails when the write lands after its pre-created names: it retries with a window sized from how long the pre-creation and write took, and where the filesystem is too slow for any window to cover the write it reports itself inconclusive and passes. The marker writer's doc says the Windows replace is untested under concurrency. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 134 +++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 44 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 27150c2d60..869a930c2f 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -849,8 +849,10 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { /// directory at once (two terminals, a script), so the write goes through the /// shared atomic writer: each writer stages into its own exclusively created /// temp file, and the marker is replaced rather than removed first, so a -/// reader always finds a complete marker. It does not order the two -/// activations: the last one to publish wins, as before. +/// reader always finds a complete marker. On Windows the replace goes through +/// `ReplaceFileW`, and two replacements in flight at once there are not +/// tested. It does not order the two activations: the last one to publish +/// wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); // The shared helper rather than a local temp-file-and-rename: it is the same @@ -1224,10 +1226,13 @@ pub(crate) mod tests { Ok(leftovers) } - /// Regression for #605: the marker used to be staged at - /// `active.json.tmp-` with a truncating write, so a writer that - /// drew the same millisecond as another activation wrote into *that* - /// writer's temp file and then moved it into place as the marker. + /// The marker is staged through `rocm_core::atomic_write`, the writer this + /// change makes `rocm` and `rocmd` share (rocmd's own copy is gone), so it + /// never writes through a temp file another writer reserved. #605 reported + /// the marker staged at `active.json.tmp-` with a truncating write, + /// where a writer that drew another activation's millisecond wrote into + /// *that* writer's temp file and moved it into place; #424 fixed that on + /// its own, and this test keeps the marker from going back to it. /// /// Pre-creating a file at every name that scheme could pick while this /// test runs stands in for the other writer deterministically: the write @@ -1237,61 +1242,102 @@ pub(crate) mod tests { /// construction, so collision safety under the current scheme is covered /// by `concurrent_active_runtime_marker_writes_keep_a_complete_marker_published` /// and by `rocm_core::atomic_write`'s own concurrency tests. + /// + /// The pre-created names only cover the milliseconds they were made for, so + /// a write that lands after them proves nothing. The window starts at 2 s + /// and, when a slow or scanned filesystem lets the write miss it, is + /// retried at four times what the pre-creation and write took (up to + /// 16 s). Where creating the names is itself slower than one per + /// millisecond no window can cover the write, and where every try misses, + /// the test says it is inconclusive on stderr and passes rather than + /// failing code that is correct. #[test] fn active_runtime_marker_write_never_reuses_another_writers_temp_file() -> Result<()> { - const WINDOW_MS: u128 = 2_000; const FOREIGN: &[u8] = b"another writer's staged marker"; + const ATTEMPTS: usize = 3; + const MAX_WINDOW_MS: u128 = 16_000; let (root, paths) = test_paths("marker-temp-collision"); write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "previous"))?; let marker_path = active_runtime_marker_path(&paths); - let start = rocm_core::unix_time_millis(); - let foreign: Vec = (start..start + WINDOW_MS) - .map(|millis| marker_path.with_extension(format!("json.tmp-{millis}"))) - .collect(); - for path in &foreign { - fs::write(path, FOREIGN)?; - } + let mut window_ms: u128 = 2_000; + for attempt in 1..=ATTEMPTS { + let start = rocm_core::unix_time_millis(); + let foreign: Vec = (start..start + window_ms) + .map(|millis| marker_path.with_extension(format!("json.tmp-{millis}"))) + .collect(); + for path in &foreign { + fs::write(path, FOREIGN)?; + } + let setup_ms = rocm_core::unix_time_millis() - start; - write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "next"))?; - let finished = rocm_core::unix_time_millis(); - assert!( - finished < start + WINDOW_MS, - "inconclusive: the write ran after the {WINDOW_MS} ms window of pre-created names" - ); + write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "next"))?; + let finished = rocm_core::unix_time_millis(); - let published: ActiveRuntimeMarker = serde_json::from_slice(&fs::read(&marker_path)?)?; - assert_eq!(published.runtime_key, "next"); - for path in &foreign { - assert_eq!( - fs::read(path).ok().as_deref(), - Some(FOREIGN), - "{} belongs to another writer but was written through or moved", - path.display() + for path in &foreign { + assert_eq!( + fs::read(path).ok().as_deref(), + Some(FOREIGN), + "{} belongs to another writer but was written through or moved", + path.display() + ); + } + for path in &foreign { + fs::remove_file(path)?; + } + if finished < start + window_ms { + let published: ActiveRuntimeMarker = + serde_json::from_slice(&fs::read(&marker_path)?)?; + assert_eq!(published.runtime_key, "next"); + assert_eq!(leftover_marker_temp_files(&paths)?, Vec::::new()); + let _ = fs::remove_dir_all(root); + return Ok(()); + } + // Missed: the names ran out before the write ran. One name covers + // one millisecond, so when creating them is slower than one per + // millisecond no window can be wide enough; otherwise size the + // next one from what this pre-creation and write took. + let next_ms = 4 * (finished - start); + if setup_ms * 2 >= window_ms || next_ms > MAX_WINDOW_MS { + eprintln!( + "inconclusive: pre-creating {window_ms} names took {setup_ms} ms and the \ + write finished after them; this filesystem is too slow for the test \ + to say anything, so it is skipped" + ); + let _ = fs::remove_dir_all(root); + return Ok(()); + } + window_ms = window_ms.max(next_ms); + eprintln!( + "attempt {attempt}/{ATTEMPTS}: the write finished after the pre-created \ + names ran out; retrying with a {window_ms} ms window" ); } - for path in &foreign { - fs::remove_file(path)?; - } - assert_eq!(leftover_marker_temp_files(&paths)?, Vec::::new()); - + eprintln!( + "inconclusive: on every attempt the write ran after the window of pre-created \ + names, so this host is too slow for this test to say anything" + ); let _ = fs::remove_dir_all(root); Ok(()) } - /// Regression for #605: two activations writing the marker at once must - /// both succeed, and a reader must find a complete marker throughout — - /// the old writer removed `active.json` before renaming the replacement - /// in, and two writers in the same millisecond shared one temp file, so - /// one failed with "failed to move active runtime marker" and the other - /// could publish the shorter payload followed by the longer one's tail. + /// Two activations writing the marker at once must both succeed, and a + /// reader must find a complete marker throughout. #605 reported a writer + /// that removed `active.json` before renaming the replacement in, and two + /// writers in the same millisecond sharing one temp file, so one failed + /// with "failed to move active runtime marker" and the other could publish + /// the shorter payload followed by the longer one's tail. #424 fixed that; + /// this change routes the marker through `rocm_core::atomic_write`, the + /// writer `rocm` and `rocmd` now share, and this test pins that the marker + /// keeps the property through it. /// /// The bug is a race, so against the old code this fails with high - /// probability per round rather than deterministically; against the fix - /// nothing in it is timing-dependent. Unix only: the property under test - /// is that `rename` replaces in one step. The Windows publish path goes - /// through `ReplaceFileW` instead, whose behaviour with two replacements - /// of one file in flight this test has not been run against. + /// probability per round rather than deterministically; against the + /// current writer nothing in it is timing-dependent. Unix only: the + /// property under test is that `rename` replaces in one step. The Windows + /// publish path goes through `ReplaceFileW` instead, whose behaviour with + /// two replacements of one file in flight this test has not been run + /// against. #[cfg(unix)] #[test] fn concurrent_active_runtime_marker_writes_keep_a_complete_marker_published() -> Result<()> { From 5141199ce5922467f0ad4164e70b944bd3c81352 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 13 Oct 2026 16:31:35 +0000 Subject: [PATCH 11/17] docs(architecture): call atomic_write the shared writer and name exceptions It is not the only temp-then-rename writer: remote/session.rs keeps its own, and RocmCliConfig::save keeps its own until #541 moves it, tracked in #618. Also say it is atomic, not durable. Signed-off-by: Roman Inflianskas --- docs/architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture.md b/docs/architecture.md index 0b5da7b030..2b7b0f2717 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -171,7 +171,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. -- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the one crash- and race-safe file replacement both `rocm` and `rocmd` use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); add a caller rather than another copy. +- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the shared atomic file writer `rocm` and `rocmd` both use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); atomic, not durable, since nothing is fsynced. Two hand-rolled temp-then-rename writers remain outside it: [`remote/session.rs`](../apps/rocm/src/remote/session.rs)'s session-file write, and rocm-core's `write_atomic` in [`lib.rs`](../crates/rocm-core/src/lib.rs), which `RocmCliConfig::save` in [`config.rs`](../crates/rocm-core/src/config.rs), the automation event and proposal stores in [`automation.rs`](../crates/rocm-core/src/automation.rs), the JSONL stores in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and the service-record manifests in [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use; it is `pub` because `rocm-engine-lemonade` also writes through it, in [`install.rs`](../engines/lemonade/src/install.rs) (see the `write_atomic` entry below). #541, which builds on this writer, moves the config save onto it; #618 tracks that move. A new atomic write should call this module rather than add another copy. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. From fe6053311d9e2616f28d17ddecc463cf77eb0f4d Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 10:27:27 +0200 Subject: [PATCH 12/17] fix(core): keep the new contents when a Windows replace fails part-way ReplaceFileW, called without a backup name, can fail after it has moved the old file off the destination (ERROR_UNABLE_TO_MOVE_REPLACEMENT and _2). The writer then deleted its temp file, the only remaining copy. On either code, retry once as a plain rename; if the destination that existed is still gone, keep the temp file and name it in the error. The Windows arm is type-checked by CI only. Also, from review: the module doc no longer claims the destination is never absent, and says which concurrency tests run on Windows; the multi-dot temp-name test captures the name at publish instead of after cleanup; the concurrent staging test forces the shared temp name so the collision happens every run; the restore path drops a create_dir_all the writer already does; a stale test-helper doc and the writer list in docs/architecture.md are corrected; the millis-only marker test is renamed to what it pins. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 24 ++-- crates/rocm-core/src/atomic_write.rs | 167 +++++++++++++++++++++++---- docs/architecture.md | 2 +- 3 files changed, 155 insertions(+), 38 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 869a930c2f..08c041f178 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -473,12 +473,8 @@ impl ActivationSnapshot { // failed, so it is the write least able to afford leaving a // truncated marker behind — which runtime resolution and the // storage retention holds would both read as "nothing is active". + // The helper creates the `runtimes` folder when it is missing. MarkerSnapshot::Contents(bytes) => { - let parent = path - .parent() - .context("active runtime marker path has no parent directory")?; - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; rocm_core::atomic_write::write_file_atomically(&path, &bytes) .with_context(|| format!("failed to restore {}", path.display()))?; } @@ -851,7 +847,9 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { /// temp file, and the marker is replaced rather than removed first, so a /// reader always finds a complete marker. On Windows the replace goes through /// `ReplaceFileW`, and two replacements in flight at once there are not -/// tested. It does not order the two activations: the last one to publish +/// tested; a `ReplaceFileW` that fails part-way can leave the marker missing, +/// which the writer repairs with a plain rename where it can (see +/// `rocm_core::atomic_write`). It does not order the two activations: the last one to publish /// wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); @@ -864,7 +862,9 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> // window with no marker at all, which runtime resolution and the storage // retention holds both read as "nothing is active" — and a failed publish // takes its temp file with it, so a retried activation cannot strand one in - // the folder `rocm runtimes` reads. It also creates the `runtimes` folder + // the folder `rocm runtimes` reads. The one exception is a Windows publish + // that took the old marker away and could not put the new one in place: + // the temp file is then the only copy, so it is kept and the error names it. It also creates the `runtimes` folder // when it is missing. let bytes = serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; @@ -1252,7 +1252,7 @@ pub(crate) mod tests { /// the test says it is inconclusive on stderr and passes rather than /// failing code that is correct. #[test] - fn active_runtime_marker_write_never_reuses_another_writers_temp_file() -> Result<()> { + fn active_runtime_marker_write_never_returns_to_millis_only_temp_names() -> Result<()> { const FOREIGN: &[u8] = b"another writer's staged marker"; const ATTEMPTS: usize = 3; const MAX_WINDOW_MS: u128 = 16_000; @@ -1537,10 +1537,10 @@ pub(crate) mod tests { } /// Replace the active-runtime marker with a NON-EMPTY directory, so the - /// next `write_active_runtime_marker` fails at a predictable point: its - /// `create_dir_all` of the parent succeeds, and the final `fs::rename` of - /// the temp file onto a non-empty directory fails on both supported - /// platforms without needing privileges or a real full disk. + /// next `write_active_runtime_marker` fails at a predictable point: the + /// temp file is staged beside it, and publishing that file onto a + /// non-empty directory fails on both supported platforms without needing + /// privileges or a real full disk. fn break_active_runtime_marker(paths: &AppPaths) -> Result { let marker = active_runtime_marker_path(paths); let _ = fs::remove_file(&marker); diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index fc60e3c3ac..edcecc6731 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -7,7 +7,7 @@ //! [`write_file_atomically`] stages the bytes in a fresh sibling temp file and //! then publishes it over the destination in one step, so a reader sees either //! the previous complete file or the new complete file, never a partial one, -//! and the destination is never absent while it is being replaced. +//! and the destination is not removed before the new file takes its place. //! //! The temp file is named `{file name}.tmp-{pid}-{millis}-{attempt}` and //! reserved with `create_new`, so two writers — threads of one process or @@ -19,9 +19,18 @@ //! `ReplaceFileW` rather than `rename`. `std::fs::rename` does replace an //! existing file there, but as `MoveFileExW` it fails with a sharing violation //! while another process holds the destination open, which is the case -//! `ReplaceFileW` is used for. Two publishes of one file in flight at once -//! through `ReplaceFileW` are not exercised by any test; the concurrency tests -//! below run on Unix only. +//! `ReplaceFileW` is used for. The concurrency tests below run on every +//! platform, but each starts with no destination, so on Windows at most one of +//! their publishes goes through `ReplaceFileW`: two `ReplaceFileW` publishes of +//! one file in flight at once are not exercised by any test. +//! +//! One failure leaves the destination absent: Microsoft documents that +//! `ReplaceFileW`, called without a backup name, can fail after it has already +//! moved the old file off the destination name +//! (`ERROR_UNABLE_TO_MOVE_REPLACEMENT` and `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`). +//! The publish then retries once as a plain rename. If the destination is +//! still missing after that, the temp file is kept rather than removed, since +//! it holds the only copy of the new contents, and the error names it. //! //! Atomic, not durable: neither the temp file nor its directory is fsynced //! before the publish. That survives a crashed process and a concurrent @@ -209,15 +218,35 @@ fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { publish_staged_file_with(tmp, path, publish_temp_file) } +/// Publish a staged file over `path`, removing the temp file if publishing +/// fails — unless the failed publish took the existing file away. +/// +/// A rename either replaces the destination or leaves it alone, but Windows' +/// `ReplaceFileW` can fail after moving the old file off the destination name +/// (see `replace_file_windows`). Removing the temp file then would leave no +/// copy of the contents under any name the user can find, so it is kept and +/// the error says where. fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> where F: FnOnce(&Path, &Path) -> io::Result<()>, { - publish(tmp, path) - .inspect_err(|_| { - let _ = fs::remove_file(tmp); - }) - .with_context(|| format!("failed to publish {}", path.display())) + let destination_existed = path.exists(); + let Err(error) = publish(tmp, path) else { + return Ok(()); + }; + if destination_existed && !path.exists() && tmp.exists() { + return Err(error).with_context(|| { + format!( + "failed to publish {}, and the replace took the old file away; the new \ + contents are kept in {} - rename that file to {} to restore it", + path.display(), + tmp.display(), + path.display() + ) + }); + } + let _ = fs::remove_file(tmp); + Err(error).with_context(|| format!("failed to publish {}", path.display())) } /// Test seam: move the staged file `tmp` over `path` in one step, replacing @@ -261,6 +290,9 @@ pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { #[allow(unsafe_code)] // Win32 FFI fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Foundation::{ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + }; use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); @@ -283,11 +315,37 @@ fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { std::ptr::null(), ) }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) + if replaced != 0 { + return Ok(()); + } + let error = io::Error::last_os_error(); + // Microsoft documents two partial outcomes for a call without a backup + // name, and in both the replacement is still under its own name: + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT` ("the replaced file no longer exists + // and the replacement file exists under its original name") and + // `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` ("the file to be replaced still + // exists with a different name"). Either way nothing is at `path` any + // more. One plain rename puts the new contents there; if that fails too, + // `publish_staged_file_with` keeps the replacement file. + let partial = [ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, + ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + ] + .into_iter() + .any(|code| { + error + .raw_os_error() + .is_some_and(|raw| u32::try_from(raw) == Ok(code)) + }); + if !partial { + return Err(error); } + fs::rename(replacement, path).map_err(|retry| { + io::Error::new( + retry.kind(), + format!("{error}; retrying as a plain rename also failed: {retry}"), + ) + }) } #[cfg(test)] @@ -346,28 +404,40 @@ mod tests { assert_eq!(temp.parent().unwrap(), Path::new("/tmp/cache")); } - /// Same property end to end: the temp file the real write creates (and - /// then cleans up) leaves only the destination behind. + /// Same property end to end: the temp file a write stages, observed at the + /// moment it is published, keeps the destination's full name, and the + /// publish leaves only the destination behind. #[test] fn write_file_atomically_temp_name_preserves_multi_dot_file_names() { let root = scratch_dir("atomic-name"); - let occupied = root.join("artifact.tar.gz"); - fs::create_dir_all(occupied.join("nested")).unwrap(); - fs::write(occupied.join("nested").join("keep"), b"x").unwrap(); + let destination = root.join("artifact.tar.gz"); + let temp_id = process_temp_id(); + let mut staged_name = None; - // Fails after the temp file exists, so the observed name is the real one. - write_file_atomically(&occupied, b"payload").expect_err("rename should fail"); + write_file_atomically_with_publish( + &destination, + b"payload", + |attempt| OsString::from(format!("{temp_id}-{attempt}")), + || {}, + |tmp, path| { + staged_name = tmp.file_name().map(OsStr::to_os_string); + publish_temp_file(tmp, path) + }, + ) + .unwrap(); let names: Vec = fs::read_dir(&root) .unwrap() .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) .collect(); + let published = fs::read(&destination); let _ = fs::remove_dir_all(&root); assert_eq!( - names, - vec!["artifact.tar.gz".to_owned()], - "only the occupied destination should remain" + staged_name, + Some(OsString::from(format!("artifact.tar.gz.tmp-{temp_id}-0"))) ); + assert_eq!(published.unwrap(), b"payload"); + assert_eq!(names, vec!["artifact.tar.gz".to_owned()]); } #[test] @@ -419,11 +489,22 @@ mod tests { let writers: Vec<_> = [b"first".as_slice(), b"second".as_slice()] .into_iter() - .map(|bytes| { + .enumerate() + .map(|(writer, bytes)| { let destination = destination.clone(); let before_publish = std::sync::Arc::clone(&before_publish); std::thread::spawn(move || { - let staged = stage_file_for_atomic_publish(&destination, bytes)?; + // Both threads ask for the same first name, as two writers + // sharing a pid and a millisecond would, so the collision + // happens every run rather than when the clock lines up. + let staged = + stage_file_for_atomic_publish_with(&destination, bytes, |attempt| { + if attempt == 0 { + OsString::from("same-millisecond") + } else { + OsString::from(format!("same-millisecond-{writer}-{attempt}")) + } + })?; before_publish.wait(); publish_staged_file(&staged, &destination) }) @@ -503,6 +584,42 @@ mod tests { assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); } + /// A publish that fails after taking the old file away — what Windows' + /// `ReplaceFileW` documents for `ERROR_UNABLE_TO_MOVE_REPLACEMENT` without + /// a backup name — must not also delete the temp file, which then holds the + /// only copy. The error names it so the user can put it back. + #[test] + fn a_publish_that_removes_the_destination_keeps_the_new_contents() { + let root = scratch_dir("replace-partial"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-{attempt}")), + || {}, + |_tmp, path| { + fs::remove_file(path).unwrap(); + Err(io::Error::other("simulated partial replace")) + }, + ) + .expect_err("the failed publish must be reported"); + let message = format!("{error:#}"); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let destination_exists = destination.exists(); + let _ = fs::remove_dir_all(&root); + assert!(!destination_exists); + assert_eq!(kept_bytes.unwrap(), b"new", "the new contents were deleted"); + assert!( + message.contains(&kept.display().to_string()), + "the error does not say where the contents are: {message}" + ); + } + #[cfg(unix)] #[test] fn atomic_temp_name_preserves_non_unicode_file_name_bytes() { diff --git a/docs/architecture.md b/docs/architecture.md index 2b7b0f2717..7d34b7e2f9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -171,7 +171,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): Already-extracted subsystem modules, following the same full-domain-extraction pattern (and several further siblings beyond these): - [`amd_smi.rs`](../crates/rocm-core/src/amd_smi.rs) — (ROCMAI-723) locating the `amd-smi` binary: `resolve_amd_smi_binary` (re-exported at the crate root) prefers the newest managed ROCm SDK runtime recorded in the registry, then the home-directory venv and legacy locations, then the bare name for a `PATH` lookup. It only reads the registry records [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) exposes and [`runtime.rs`](../crates/rocm-core/src/runtime.rs)'s data/home directory lookups. As of EAI-8457 it also owns the *launch*: `amd_smi_json` (also re-exported at the crate root) is the one supported way for a synchronous caller to run `amd-smi`, because it is the only place the mandatory `/dev/kfd` pre-flight runs (`KFD_DEVICE`, `preflight_kfd_device`, `kfd_readable`/`kfd_readable_with`, and the thread-bounded, budget-capped `device_readable`/`device_readable_within_budget` probe behind `KFD_OPEN_TIMEOUT`). A device that cannot be opened declines the spawn rather than letting `amd-smi` park in uninterruptible sleep, where no timeout can reach it. The spawn itself still goes through [`process.rs`](../crates/rocm-core/src/process.rs)'s `run_with_timeout`, and the non-zero-exit quote is bounded with that module's `MAX_QUOTED_OUTPUT_CHARS` rather than a second copy of the constant. Callers: [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s `gpu_vram_usage_amd_smi`/`detect_gpu_count`, both through that module's own `amd_smi_probe_json` (which logs the discarded decline at `info`, since the default filter is `warn,rocm=info` and a decline on a healthy host is otherwise indistinguishable from "no GPU"), and `rocmd`'s [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs)`::capture_amd_smi_json`. That the list stays complete is enforced rather than asserted: [`amd_smi_launch_contract.rs`](../xtask/src/amd_smi_launch_contract.rs) is a static contract test failing any file outside its two-entry allowlist that calls `resolve_amd_smi_binary`, so a new probe cannot be spawned directly without a reviewed exemption. The `rocm dash` collectors keep their own pre-flight *decision* (`preflight_passes`), since they gate an async launch, but share this module's bounded device probe through the `pub fn device_readable_bounded` re-export — an allowlisted `rocm-dash-collectors -> rocm-core` edge recorded in [`crate_edges.rs`](../xtask/src/crate_edges.rs), so the one launch site outside this chokepoint cannot drift away from the bound and the budget. -- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the shared atomic file writer `rocm` and `rocmd` both use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); atomic, not durable, since nothing is fsynced. Two hand-rolled temp-then-rename writers remain outside it: [`remote/session.rs`](../apps/rocm/src/remote/session.rs)'s session-file write, and rocm-core's `write_atomic` in [`lib.rs`](../crates/rocm-core/src/lib.rs), which `RocmCliConfig::save` in [`config.rs`](../crates/rocm-core/src/config.rs), the automation event and proposal stores in [`automation.rs`](../crates/rocm-core/src/automation.rs), the JSONL stores in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and the service-record manifests in [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use; it is `pub` because `rocm-engine-lemonade` also writes through it, in [`install.rs`](../engines/lemonade/src/install.rs) (see the `write_atomic` entry below). #541, which builds on this writer, moves the config save onto it; #618 tracks that move. A new atomic write should call this module rather than add another copy. +- [`atomic_write.rs`](../crates/rocm-core/src/atomic_write.rs) — the shared atomic file writer `rocm` and `rocmd` both use (unique exclusive temp file, then rename, or `ReplaceFileW` on Windows); atomic, not durable, since nothing is fsynced. Other hand-rolled temp-then-rename writers remain outside it, for example [`remote/session.rs`](../apps/rocm/src/remote/session.rs)'s session-file write, [`uninstall.rs`](../apps/rocm/src/uninstall.rs)'s profile replacement, `write_atomically` in [`comfyui/source_swap.rs`](../apps/rocm/src/comfyui/source_swap.rs), and rocm-core's `write_atomic` in [`lib.rs`](../crates/rocm-core/src/lib.rs), which `RocmCliConfig::save` in [`config.rs`](../crates/rocm-core/src/config.rs), the automation event and proposal stores in [`automation.rs`](../crates/rocm-core/src/automation.rs), the JSONL stores in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) and the service-record manifests in [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use; it is `pub` because `rocm-engine-lemonade` also writes through it, in [`install.rs`](../engines/lemonade/src/install.rs) (see the `write_atomic` entry below). #541, which builds on this writer, moves the config save onto it; #618 tracks that move. A new atomic write should call this module rather than add another copy. - [`automation.rs`](../crates/rocm-core/src/automation.rs) — (ROCMAI-151) automation-daemon runtime state and its on-disk event/proposal/audit records, and the policy applied to each store (which are rotated, and the proposal compaction) — see the module's own doc comment for what it owns. The generic JSONL machinery under those stores lives in [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs). The dependency runs one way: this module reads `WatcherMode` from [`config.rs`](../crates/rocm-core/src/config.rs), and [`config.rs`](../crates/rocm-core/src/config.rs) imports nothing from here, because the built-in watcher catalog (whose entries carry a default `WatcherMode`, and which `RocmCliConfig::watcher_enabled` and `effective_watcher_mode` take as an argument) lives in [`config.rs`](../crates/rocm-core/src/config.rs) beside `WatcherMode`. - [`binary_paths.rs`](../crates/rocm-core/src/binary_paths.rs) — (ROCMAI-151, Phase 7d) locating `rocm-engine-` siblings, and the CLI binary itself (a sibling `rocm` only when running from a cargo `deps/` directory). - [`codex_bridge.rs`](../crates/rocm-core/src/codex_bridge.rs) — (ROCMAI-662) the `CodexBridgeSnapshot`/`CodexBridgeGpuSnapshot`/`CodexBridgeEngine` host-snapshot DTOs. Plain serde data: `rocm` and `rocmd` each build their own snapshot of this shape, so the types live here rather than in either binary. From 9eab4c86c12841a02c394ce6812ba20f0c71d8d8 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 5 Nov 2026 16:22:20 +0200 Subject: [PATCH 13/17] fix(core): decide a kept replacement from the error, not a stale look A publish kept its temp file only when the destination had existed before the publish and was gone after it. A second writer that created the destination in between made that pre-publish look stale, and the temp file holding the only copy was deleted. The Windows partial-replace path now reports itself: when ReplaceFileW fails with ERROR_UNABLE_TO_MOVE_REPLACEMENT or _2 and the retried rename fails too, the publish returns a typed error, and the writer keeps the temp file on that error alone, returning a KeptReplacement that names it. The code check and retry are a plain function over the error code and an injected rename, so they run as unit tests on every platform; a Windows-only test ties the codes to windows-sys. A failed activation rolls the marker back, so the kept file then holds the marker of the activation that failed, and the advice to rename it into place would install that marker over the restored one. The rollback now removes it and rebuilds the error without that advice, on both the restored and the failed-restore path; a file it cannot remove is named with advice to delete it instead. The marker writer's doc no longer says a reader always finds a complete marker without naming the Windows exception. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 248 ++++++++++++++++++- crates/rocm-core/src/atomic_write.rs | 353 +++++++++++++++++++++++---- 2 files changed, 547 insertions(+), 54 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 08c041f178..9c5539693c 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -523,11 +523,11 @@ fn restore_after_failed_activation( error: anyhow::Error, ) -> anyhow::Error { match snapshot.restore(paths, config) { - Ok(()) => error.context( + Ok(()) => discard_failed_marker_replacement(error).context( "failed to record the active ROCm runtime; the previously active runtime is still \ the active one", ), - Err(restore_error) => error.context(format!( + Err(restore_error) => discard_failed_marker_replacement(error).context(format!( "failed to record the active ROCm runtime, and restoring the previous state failed \ too ({restore_error:#}); the ROCm CLI config and the active runtime marker may now \ disagree. Run `rocm runtimes list` to see both, then re-run `rocm runtimes activate \ @@ -536,6 +536,38 @@ fn restore_after_failed_activation( } } +/// Drop the marker a failed activation left behind, and the advice to use it. +/// +/// On Windows a marker publish can fail after taking the old marker away, and +/// the shared writer then keeps the new marker in its temp file and advises +/// renaming it into place (`rocm_core::atomic_write::KeptReplacement`). The +/// activation is being rolled back, so that file holds the marker of an +/// activation that failed, and renaming it would install that marker over +/// whatever the rollback restored. So the file is removed, and the error is +/// rebuilt without the advice. If the file cannot be removed, the error names +/// it and says to delete it instead. +fn discard_failed_marker_replacement(error: anyhow::Error) -> anyhow::Error { + let Some(kept) = error.downcast_ref::() else { + return error; + }; + let failure = format!( + "failed to write active runtime marker {}: {}", + kept.destination().display(), + kept.cause() + ); + match fs::remove_file(kept.kept()) { + Ok(()) => anyhow::anyhow!(failure), + Err(remove_error) if remove_error.kind() == std::io::ErrorKind::NotFound => { + anyhow::anyhow!(failure) + } + Err(remove_error) => anyhow::anyhow!( + "{failure}; {} holds the marker of the activation that failed and could not be \ + removed ({remove_error}): delete it, do not rename it into place", + kept.kept().display() + ), + } +} + /// Marker shown beside the active runtime in `rocm runtimes list`. Every /// place that renders this glyph MUST use this constant so the rendered /// character and the legend text stay in sync. @@ -844,13 +876,13 @@ pub(crate) const fn toolchain_state_text(includes_devel: bool) -> &'static str { /// Two `rocm` processes can activate a runtime against the same data /// directory at once (two terminals, a script), so the write goes through the /// shared atomic writer: each writer stages into its own exclusively created -/// temp file, and the marker is replaced rather than removed first, so a -/// reader always finds a complete marker. On Windows the replace goes through -/// `ReplaceFileW`, and two replacements in flight at once there are not -/// tested; a `ReplaceFileW` that fails part-way can leave the marker missing, -/// which the writer repairs with a plain rename where it can (see -/// `rocm_core::atomic_write`). It does not order the two activations: the last one to publish -/// wins, as before. +/// temp file, and the marker is replaced rather than removed first. On Unix, +/// and on Windows whenever `ReplaceFileW` succeeds, a reader therefore always +/// finds a complete marker. A `ReplaceFileW` that fails part-way is the +/// exception: it can leave the marker missing, which the writer repairs with a +/// plain rename when that rename succeeds (see `rocm_core::atomic_write`); two +/// replacements in flight at once on Windows are not tested. It does not order +/// the two activations: the last one to publish wins, as before. fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> Result<()> { let path = active_runtime_marker_path(paths); // The shared helper rather than a local temp-file-and-rename: it is the same @@ -864,8 +896,10 @@ fn write_active_runtime_marker(paths: &AppPaths, marker: ActiveRuntimeMarker) -> // takes its temp file with it, so a retried activation cannot strand one in // the folder `rocm runtimes` reads. The one exception is a Windows publish // that took the old marker away and could not put the new one in place: - // the temp file is then the only copy, so it is kept and the error names it. It also creates the `runtimes` folder - // when it is missing. + // the temp file is then the only copy, so it is kept and the error names + // it; a failed activation removes it again once the restore has put the + // previous marker back (`discard_failed_marker_replacement`). It also + // creates the `runtimes` folder when it is missing. let bytes = serde_json::to_vec_pretty(&marker).context("failed to serialize active runtime marker")?; rocm_core::atomic_write::write_file_atomically(&path, &bytes) @@ -2232,6 +2266,198 @@ pub(crate) mod tests { Ok(()) } + /// The Windows partial-replace failure during activation: the marker publish + /// took the old marker away and kept the new one in its temp file. Once the + /// restore has put the previous marker back, the kept file holds the failed + /// activation's marker, so it is removed, and the error no longer tells the + /// user to rename it into place. Driven through the shared writer's publish + /// seam, so it runs on every platform. + #[test] + fn a_restored_activation_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.previous_runtime_key = Some(OLD_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + + let marker_path = active_runtime_marker_path(&paths); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!( + !marker_temp_file_names(&paths)?.is_empty(), + "the shared writer keeps the new marker when the old one is gone" + ); + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(&marker_path)?; + let leftovers = marker_temp_file_names(&paths)?; + let persisted = RocmCliConfig::load(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(restored_marker, previous_marker); + assert_eq!( + persisted.active_runtime_key.as_deref(), + Some(OLD_RUNTIME_KEY) + ); + assert!( + leftovers.is_empty(), + "the failed activation's marker was left beside the restored one: {leftovers:?}" + ); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// Produce the error the shared writer returns when a marker publish took + /// the old marker away and kept the new one, through its publish seam. + /// `prepare_kept` runs on the kept temp file before the error is returned. + fn partial_replace_marker_error( + paths: &AppPaths, + prepare_kept: impl FnOnce(&std::path::Path) -> std::io::Result<()>, + ) -> anyhow::Error { + let marker_path = active_runtime_marker_path(paths); + rocm_core::atomic_write::write_file_atomically_with_publish( + &marker_path, + b"{\"runtime_key\":\"new\"}", + |attempt| std::ffi::OsString::from(format!("partial-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path)?; + prepare_kept(tmp)?; + Err(rocm_core::atomic_write::destination_removed_error( + std::io::Error::from_raw_os_error(1176), + std::io::Error::from(std::io::ErrorKind::PermissionDenied), + )) + }, + ) + .expect_err("the simulated partial replace fails the write") + .context(format!( + "failed to write active runtime marker {}", + marker_path.display() + )) + } + + /// When the kept marker cannot be removed, the error names it and says to + /// delete it, and nothing in it still says to rename it into place. The + /// kept path is made a non-empty directory, which `remove_file` refuses on + /// every platform. + #[test] + fn a_kept_marker_that_cannot_be_removed_is_named_with_delete_advice_only() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-stuck"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + let previous_marker = fs::read(active_runtime_marker_path(&paths))?; + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + config.save(&paths)?; + let write_error = partial_replace_marker_error(&paths, |tmp| { + fs::remove_file(tmp)?; + fs::create_dir(tmp)?; + fs::write(tmp.join("occupied.txt"), "stuck") + }); + let kept = marker_temp_file_names(&paths)?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let restored_marker = fs::read(active_runtime_marker_path(&paths))?; + let still_there = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert_eq!(still_there, kept, "the stuck file is left where it was"); + assert_eq!(restored_marker, previous_marker); + assert!( + rendered.contains(&kept[0]), + "the stuck file is named:\n{rendered}" + ); + assert!( + rendered.contains("delete it, do not rename it"), + "{rendered}" + ); + assert!( + !rendered.contains("rename that file"), + "renaming would install the failed activation's marker:\n{rendered}" + ); + Ok(()) + } + + /// A kept marker that is already gone by the time the rollback looks for it + /// is as good as removed: the error neither names it nor tells the user to + /// delete or rename it. + #[test] + fn a_kept_marker_already_gone_is_treated_as_removed() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-gone"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + let kept = marker_temp_file_names(&paths)?; + let runtimes_dir = active_runtime_marker_path(&paths) + .parent() + .context("active runtime marker path has no parent directory")? + .to_path_buf(); + for name in &kept { + fs::remove_file(runtimes_dir.join(name))?; + } + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let _ = fs::remove_dir_all(&root); + + assert_eq!(kept.len(), 1, "{kept:?}"); + assert!( + rendered.contains("the previously active runtime is still the active one"), + "{rendered}" + ); + assert!(!rendered.contains(&kept[0]), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(!rendered.contains("delete it"), "{rendered}"); + Ok(()) + } + + /// When the rollback itself fails as well, the kept marker is still the + /// failed activation's, so it goes too, and the error keeps the rollback's + /// own recovery advice without the advice to rename it. The config path is + /// made a non-empty directory so the restore's config save fails. + #[test] + fn a_failed_rollback_still_drops_the_marker_a_partial_replace_kept() -> Result<()> { + let (root, paths) = test_paths("activation-partial-replace-double-fault"); + paths.ensure()?; + let mut config = activate_old_runtime(&paths)?; + let snapshot = ActivationSnapshot::capture(&paths, &config); + config.active_runtime_key = Some(NEW_RUNTIME_KEY.to_owned()); + let write_error = partial_replace_marker_error(&paths, |_| Ok(())); + assert!(!marker_temp_file_names(&paths)?.is_empty()); + let config_path = paths.config_path(); + let _ = fs::remove_file(&config_path); + fs::create_dir_all(&config_path)?; + fs::write(config_path.join("occupied.txt"), "not a config")?; + + let error = restore_after_failed_activation(&paths, &mut config, snapshot, write_error); + let rendered = format!("{error:#}"); + let leftovers = marker_temp_file_names(&paths)?; + let _ = fs::remove_dir_all(&root); + + assert!( + rendered.contains("restoring the previous state failed too"), + "{rendered}" + ); + assert!(rendered.contains("rocm runtimes list"), "{rendered}"); + assert!(!rendered.contains("rename that file"), "{rendered}"); + assert!(leftovers.is_empty(), "{leftovers:?}"); + Ok(()) + } + // A restore that fails too must say so and name the way out, rather than // claim the previous runtime is still the active one. The marker's parent is // made a regular file, so writing the captured bytes back fails at diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index edcecc6731..f7cb4f6f7a 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -13,7 +13,8 @@ //! reserved with `create_new`, so two writers — threads of one process or //! separate processes — can never write through the same temp file, even when //! their pid and millisecond collide; a collision just moves on to the next -//! attempt. A failed write or publish removes its temp file. +//! attempt. A failed write or publish removes its temp file, with the one +//! exception described below. //! //! On Windows, when the destination already exists, the publish uses //! `ReplaceFileW` rather than `rename`. `std::fs::rename` does replace an @@ -28,9 +29,12 @@ //! `ReplaceFileW`, called without a backup name, can fail after it has already //! moved the old file off the destination name //! (`ERROR_UNABLE_TO_MOVE_REPLACEMENT` and `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2`). -//! The publish then retries once as a plain rename. If the destination is -//! still missing after that, the temp file is kept rather than removed, since -//! it holds the only copy of the new contents, and the error names it. +//! The publish then retries once as a plain rename. If that fails too, the +//! temp file is kept rather than removed, since it holds the only copy of the +//! new contents, and the error is a [`KeptReplacement`] naming it. The +//! decision rests on the error code the replace returned, not on whether the +//! destination existed beforehand, so a second writer that created the +//! destination in between cannot turn it into a deletion. //! //! Atomic, not durable: neither the temp file nor its directory is fsynced //! before the publish. That survives a crashed process and a concurrent @@ -45,12 +49,136 @@ use crate::disk_space; use crate::unix_time_millis; use anyhow::{Context, Result, bail}; use std::ffi::{OsStr, OsString}; +use std::fmt; use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; +/// The raw values of Windows' `ERROR_UNABLE_TO_MOVE_REPLACEMENT` (1176) and +/// `ERROR_UNABLE_TO_MOVE_REPLACEMENT_2` (1177), the two codes with which +/// `ReplaceFileW` reports that it moved the old file off the destination name +/// and then failed. Plain numbers so the decision built on them can be tested +/// on every platform; a Windows-only test ties them to `windows-sys`. +#[cfg(any(windows, test))] +const PARTIAL_REPLACE_ERROR_CODES: [i32; 2] = [1176, 1177]; + +/// Whether a failed `ReplaceFileW` left the destination name empty. +#[cfg(any(windows, test))] +fn is_partial_replace(error: &io::Error) -> bool { + error + .raw_os_error() + .is_some_and(|code| PARTIAL_REPLACE_ERROR_CODES.contains(&code)) +} + +/// A publish that took the old destination away and could not put the new +/// file in its place: the replace failed part-way, and so did the rename +/// retried after it. +#[derive(Debug)] +struct DestinationRemoved { + replace: io::Error, + retry: io::Error, +} + +impl fmt::Display for DestinationRemoved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}; retrying as a plain rename also failed: {}", + self.replace, self.retry + ) + } +} + +impl std::error::Error for DestinationRemoved {} + +/// Test seam: the error a publish returns when it took the old destination +/// away and could not put the new file in its place, which is what Windows' +/// partial `ReplaceFileW` followed by a failed rename produces. +/// +/// `pub` only so tests in other crates can drive that path through +/// [`write_file_atomically_with_publish`] on any platform; production code +/// never builds one. +#[doc(hidden)] +pub fn destination_removed_error(replace: io::Error, retry: io::Error) -> io::Error { + io::Error::new(retry.kind(), DestinationRemoved { replace, retry }) +} + +fn removed_the_destination(error: &io::Error) -> bool { + error + .get_ref() + .is_some_and(::is::) +} + +/// The error of a publish that took the old file away and kept the new +/// contents in the temp file, which then holds the only copy. +/// +/// A caller that knows better than to want those contents back — one that has +/// already put the previous file back, say — can downcast to this to find the +/// kept file and remove it. +#[derive(Debug)] +pub struct KeptReplacement { + destination: PathBuf, + kept: PathBuf, + cause: io::Error, +} + +impl KeptReplacement { + /// The path the publish was meant to replace, which is now missing. + pub fn destination(&self) -> &Path { + &self.destination + } + + /// The temp file holding the new contents. + pub fn kept(&self) -> &Path { + &self.kept + } + + /// Why the publish failed. + pub const fn cause(&self) -> &io::Error { + &self.cause + } +} + +impl fmt::Display for KeptReplacement { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "failed to publish {}, and the replace took the old file away; the new contents \ + are kept in {} - rename that file to {} to restore it", + self.destination.display(), + self.kept.display(), + self.destination.display() + ) + } +} + +impl std::error::Error for KeptReplacement { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.cause) + } +} + +/// What to do after `ReplaceFileW` failed with `error`: give up when the +/// destination is untouched, and otherwise put the replacement in place with +/// `rename`, which is a plain rename in production and injected by tests. +#[cfg(any(windows, test))] +fn recover_partial_replace( + error: io::Error, + replacement: &Path, + path: &Path, + rename: R, +) -> io::Result<()> +where + R: FnOnce(&Path, &Path) -> io::Result<()>, +{ + if !is_partial_replace(&error) { + return Err(error); + } + rename(replacement, path).map_err(|retry| destination_removed_error(error, retry)) +} + /// A temp path next to `path`, preserving the full file name so a /// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes /// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). @@ -223,27 +351,25 @@ fn publish_staged_file(tmp: &Path, path: &Path) -> Result<()> { /// /// A rename either replaces the destination or leaves it alone, but Windows' /// `ReplaceFileW` can fail after moving the old file off the destination name -/// (see `replace_file_windows`). Removing the temp file then would leave no +/// (see `recover_partial_replace`). Removing the temp file then would leave no /// copy of the contents under any name the user can find, so it is kept and -/// the error says where. +/// the error is a [`KeptReplacement`] that says where. The publish step itself +/// reports that case, so it is told apart by the error rather than by a look +/// at the destination taken before the publish, which another writer can make +/// stale. fn publish_staged_file_with(tmp: &Path, path: &Path, publish: F) -> Result<()> where F: FnOnce(&Path, &Path) -> io::Result<()>, { - let destination_existed = path.exists(); let Err(error) = publish(tmp, path) else { return Ok(()); }; - if destination_existed && !path.exists() && tmp.exists() { - return Err(error).with_context(|| { - format!( - "failed to publish {}, and the replace took the old file away; the new \ - contents are kept in {} - rename that file to {} to restore it", - path.display(), - tmp.display(), - path.display() - ) - }); + if removed_the_destination(&error) && tmp.exists() { + return Err(anyhow::Error::new(KeptReplacement { + destination: path.to_path_buf(), + kept: tmp.to_path_buf(), + cause: error, + })); } let _ = fs::remove_file(tmp); Err(error).with_context(|| format!("failed to publish {}", path.display())) @@ -290,9 +416,6 @@ pub fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { #[allow(unsafe_code)] // Win32 FFI fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Foundation::{ - ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, - }; use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); @@ -318,7 +441,6 @@ fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { if replaced != 0 { return Ok(()); } - let error = io::Error::last_os_error(); // Microsoft documents two partial outcomes for a call without a backup // name, and in both the replacement is still under its own name: // `ERROR_UNABLE_TO_MOVE_REPLACEMENT` ("the replaced file no longer exists @@ -327,25 +449,14 @@ fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { // exists with a different name"). Either way nothing is at `path` any // more. One plain rename puts the new contents there; if that fails too, // `publish_staged_file_with` keeps the replacement file. - let partial = [ - ERROR_UNABLE_TO_MOVE_REPLACEMENT, - ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, - ] - .into_iter() - .any(|code| { - error - .raw_os_error() - .is_some_and(|raw| u32::try_from(raw) == Ok(code)) - }); - if !partial { - return Err(error); - } - fs::rename(replacement, path).map_err(|retry| { - io::Error::new( - retry.kind(), - format!("{error}; retrying as a plain rename also failed: {retry}"), - ) - }) + recover_partial_replace(io::Error::last_os_error(), replacement, path, rename_file) +} + +/// `fs::rename` with both paths borrowed, the shape `recover_partial_replace` +/// takes. +#[cfg(windows)] +fn rename_file(from: &Path, to: &Path) -> io::Result<()> { + fs::rename(from, to) } #[cfg(test)] @@ -584,10 +695,18 @@ mod tests { assert_eq!(leftovers, vec![OsString::from("sdk.tar.gz")]); } + fn simulated_partial_replace() -> io::Error { + destination_removed_error( + io::Error::from_raw_os_error(PARTIAL_REPLACE_ERROR_CODES[0]), + io::Error::from(io::ErrorKind::PermissionDenied), + ) + } + /// A publish that fails after taking the old file away — what Windows' /// `ReplaceFileW` documents for `ERROR_UNABLE_TO_MOVE_REPLACEMENT` without - /// a backup name — must not also delete the temp file, which then holds the - /// only copy. The error names it so the user can put it back. + /// a backup name, when the rename retried after it fails too — must not + /// also delete the temp file, which then holds the only copy. The error + /// names it so the user can put it back, and a caller can find it. #[test] fn a_publish_that_removes_the_destination_keeps_the_new_contents() { let root = scratch_dir("replace-partial"); @@ -602,11 +721,14 @@ mod tests { || {}, |_tmp, path| { fs::remove_file(path).unwrap(); - Err(io::Error::other("simulated partial replace")) + Err(simulated_partial_replace()) }, ) .expect_err("the failed publish must be reported"); let message = format!("{error:#}"); + let reported = error + .downcast_ref::() + .map(|kept| (kept.kept().to_path_buf(), kept.destination().to_path_buf())); let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-0")).unwrap(); let kept_bytes = fs::read(&kept); @@ -618,6 +740,151 @@ mod tests { message.contains(&kept.display().to_string()), "the error does not say where the contents are: {message}" ); + assert_eq!(reported, Some((kept, destination))); + } + + /// The race a pre-publish look at the destination gets wrong: no + /// destination when this writer starts, another writer publishes one, and + /// then this writer's replace takes it away and fails. The temp file still + /// holds the only copy, so it is still kept. + #[test] + fn a_publish_that_removes_a_destination_another_writer_created_keeps_the_new_contents() { + let root = scratch_dir("replace-partial-race"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"second", + |attempt| OsString::from(format!("replace-partial-race-{attempt}")), + || {}, + |_tmp, path| { + fs::write(path, b"first").unwrap(); + fs::remove_file(path).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + + let kept = temp_sibling_path(&destination, OsStr::new("replace-partial-race-0")).unwrap(); + let kept_bytes = fs::read(&kept); + let _ = fs::remove_dir_all(&root); + assert_eq!( + kept_bytes.unwrap(), + b"second", + "the new contents were deleted" + ); + assert!( + error.downcast_ref::().is_some(), + "{error:#}" + ); + } + + /// When the temp file is gone too there is nothing to keep, so the error + /// must not point the user at it. + #[test] + fn a_partial_replace_that_lost_the_temp_file_does_not_claim_to_keep_it() { + let root = scratch_dir("replace-partial-no-temp"); + let destination = root.join("active.json"); + fs::create_dir_all(&root).unwrap(); + fs::write(&destination, b"old").unwrap(); + + let error = write_file_atomically_with_publish( + &destination, + b"new", + |attempt| OsString::from(format!("replace-partial-no-temp-{attempt}")), + || {}, + |tmp, path| { + fs::remove_file(path).unwrap(); + fs::remove_file(tmp).unwrap(); + Err(simulated_partial_replace()) + }, + ) + .expect_err("the failed publish must be reported"); + let _ = fs::remove_dir_all(&root); + let message = format!("{error:#}"); + assert!( + error.downcast_ref::().is_none(), + "{message}" + ); + assert!(!message.contains("kept in"), "{message}"); + } + + /// Any other failed replace leaves the destination alone, so nothing is + /// retried and the error comes back as it was. + #[test] + fn a_replace_that_left_the_destination_alone_is_not_retried() { + let mut renamed = false; + let error = recover_partial_replace( + io::Error::from_raw_os_error(5), + Path::new("staged"), + Path::new("destination"), + |_, _| { + renamed = true; + Ok(()) + }, + ) + .expect_err("an ordinary failure is returned"); + assert!(!renamed); + assert_eq!(error.raw_os_error(), Some(5)); + assert!(!removed_the_destination(&error)); + } + + /// Both documented partial codes retry once as a rename of the staged + /// file onto the destination, and a successful retry is a success. + #[test] + fn a_partial_replace_is_retried_as_a_rename() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let mut renamed = None; + recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |from, to| { + renamed = Some((from.to_path_buf(), to.to_path_buf())); + Ok(()) + }, + ) + .unwrap_or_else(|error| panic!("code {code}: {error}")); + assert_eq!( + renamed, + Some((PathBuf::from("staged"), PathBuf::from("destination"))), + "code {code}" + ); + } + } + + /// A retry that fails too is reported as the case that keeps the temp file. + #[test] + fn a_partial_replace_whose_retry_fails_reports_the_destination_removed() { + for code in PARTIAL_REPLACE_ERROR_CODES { + let error = recover_partial_replace( + io::Error::from_raw_os_error(code), + Path::new("staged"), + Path::new("destination"), + |_, _| Err(io::Error::from(io::ErrorKind::PermissionDenied)), + ) + .expect_err("a failed retry is returned"); + assert!(removed_the_destination(&error), "code {code}: {error}"); + assert!(error.to_string().contains("retrying as a plain rename")); + } + } + + /// The plain numbers the partial-replace decision uses are the codes + /// `ReplaceFileW` actually returns. + #[cfg(windows)] + #[test] + fn partial_replace_codes_match_the_windows_constants() { + use windows_sys::Win32::Foundation::{ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, ERROR_UNABLE_TO_MOVE_REPLACEMENT_2, + }; + assert_eq!( + PARTIAL_REPLACE_ERROR_CODES.map(|code| u32::try_from(code).unwrap()), + [ + ERROR_UNABLE_TO_MOVE_REPLACEMENT, + ERROR_UNABLE_TO_MOVE_REPLACEMENT_2 + ] + ); } #[cfg(unix)] From 08e3db3a767a9eaf81393adb23da38fea5ba4058 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 5 Nov 2026 16:23:43 +0200 Subject: [PATCH 14/17] test(core): cover the writer giving up when every temp name is taken After 128 collisions the stager fails with an error that says so. No test reached that branch; this one makes every name collide and checks the bounded number of tries, the error, and that the other writer's file is left as it was. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/atomic_write.rs | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index f7cb4f6f7a..c54c9c6c7a 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -504,6 +504,40 @@ mod tests { ); } + /// When every temp name the writer tries is taken, it gives up after a + /// bounded number of tries with an error that says so, rather than + /// looping or writing through a file another writer reserved. + #[test] + fn staging_gives_up_when_every_temp_name_is_taken() { + let root = scratch_dir("atomic-names-exhausted"); + fs::create_dir_all(&root).unwrap(); + let destination = root.join("active.json"); + let taken = temp_sibling_path(&destination, OsStr::new("taken")).unwrap(); + fs::write(&taken, b"another writer's staged file").unwrap(); + let mut tries = 0; + + let error = stage_file_for_atomic_publish_with(&destination, b"payload", |_| { + tries += 1; + OsString::from("taken") + }) + .expect_err("a writer that finds every name taken must fail"); + + let foreign = fs::read(&taken); + let leftovers: Vec<_> = fs::read_dir(&root) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + let _ = fs::remove_dir_all(&root); + let text = format!("{error:#}"); + assert_eq!(tries, ATOMIC_WRITE_TEMP_ATTEMPTS); + assert!( + text.contains(&format!("after {ATOMIC_WRITE_TEMP_ATTEMPTS} attempts")), + "{text}" + ); + assert_eq!(foreign.unwrap(), b"another writer's staged file"); + assert_eq!(leftovers, vec![OsString::from("active.json.tmp-taken")]); + } + /// The temp name keeps every extension, so a cleanup sweep over a cache /// directory can still tell what a leftover was going to be. #[test] From 0f9f4c895ef3cae9ec408251f10a926e045a19b5 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Thu, 5 Nov 2026 16:23:43 +0200 Subject: [PATCH 15/17] fix(core): name the destination, not the temp file, on a full disk A write that ran out of space named the staged temp file, which the writer has already removed by the time the user reads the message. The destination is the file they asked for and the one on the filesystem they need to free space on. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/atomic_write.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/atomic_write.rs b/crates/rocm-core/src/atomic_write.rs index c54c9c6c7a..a3899fde64 100644 --- a/crates/rocm-core/src/atomic_write.rs +++ b/crates/rocm-core/src/atomic_write.rs @@ -335,7 +335,7 @@ where if let Err(error) = write(&mut file, bytes) { drop(file); let _ = fs::remove_file(&tmp); - return Err(disk_space::map_write_error(error, &tmp)); + return Err(disk_space::map_write_error(error, path)); } drop(file); Ok(tmp) @@ -498,6 +498,12 @@ mod tests { let _ = fs::remove_dir_all(&root); let text = format!("{error:#}"); assert!(text.contains("ran out of disk space"), "{text}"); + // The user knows the file they asked for, not the temp file that was + // already removed, so the message names the destination. + assert!( + text.contains(&destination.display().to_string()) && !text.contains(".tmp-"), + "{text}" + ); assert!( leftovers.is_empty(), "the failed staging left files behind: {leftovers:?}" From 4be5acc4704e7731d81b605861aadc38429bacbe Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 6 Nov 2026 17:11:42 +0200 Subject: [PATCH 16/17] test(rocm): drop the millis-only temp-name marker test The test pre-created files at every name the retired millis-only scheme could pick, then checked the marker write left them alone. When the write missed that window it reported "inconclusive" and passed, after creating up to thousands of files. It also guarded a naming scheme no longer in the tree, since the marker now goes through rocm_core::atomic_write. concurrent_active_runtime_marker_writes_keep_a_complete_marker_published and the writer's own collision tests carry the guarantee. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 95 ----------------------------------- 1 file changed, 95 deletions(-) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 9c5539693c..74cadde934 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -1260,101 +1260,6 @@ pub(crate) mod tests { Ok(leftovers) } - /// The marker is staged through `rocm_core::atomic_write`, the writer this - /// change makes `rocm` and `rocmd` share (rocmd's own copy is gone), so it - /// never writes through a temp file another writer reserved. #605 reported - /// the marker staged at `active.json.tmp-` with a truncating write, - /// where a writer that drew another activation's millisecond wrote into - /// *that* writer's temp file and moved it into place; #424 fixed that on - /// its own, and this test keeps the marker from going back to it. - /// - /// Pre-creating a file at every name that scheme could pick while this - /// test runs stands in for the other writer deterministically: the write - /// has to succeed without touching any of them. This guards against a - /// return to that millis-only scheme specifically. The shared writer's own - /// names (`{name}.tmp-{pid}-{millis}-{attempt}`) cannot match these by - /// construction, so collision safety under the current scheme is covered - /// by `concurrent_active_runtime_marker_writes_keep_a_complete_marker_published` - /// and by `rocm_core::atomic_write`'s own concurrency tests. - /// - /// The pre-created names only cover the milliseconds they were made for, so - /// a write that lands after them proves nothing. The window starts at 2 s - /// and, when a slow or scanned filesystem lets the write miss it, is - /// retried at four times what the pre-creation and write took (up to - /// 16 s). Where creating the names is itself slower than one per - /// millisecond no window can cover the write, and where every try misses, - /// the test says it is inconclusive on stderr and passes rather than - /// failing code that is correct. - #[test] - fn active_runtime_marker_write_never_returns_to_millis_only_temp_names() -> Result<()> { - const FOREIGN: &[u8] = b"another writer's staged marker"; - const ATTEMPTS: usize = 3; - const MAX_WINDOW_MS: u128 = 16_000; - let (root, paths) = test_paths("marker-temp-collision"); - write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "previous"))?; - let marker_path = active_runtime_marker_path(&paths); - - let mut window_ms: u128 = 2_000; - for attempt in 1..=ATTEMPTS { - let start = rocm_core::unix_time_millis(); - let foreign: Vec = (start..start + window_ms) - .map(|millis| marker_path.with_extension(format!("json.tmp-{millis}"))) - .collect(); - for path in &foreign { - fs::write(path, FOREIGN)?; - } - let setup_ms = rocm_core::unix_time_millis() - start; - - write_active_runtime_marker(&paths, test_active_runtime_marker(&paths, "next"))?; - let finished = rocm_core::unix_time_millis(); - - for path in &foreign { - assert_eq!( - fs::read(path).ok().as_deref(), - Some(FOREIGN), - "{} belongs to another writer but was written through or moved", - path.display() - ); - } - for path in &foreign { - fs::remove_file(path)?; - } - if finished < start + window_ms { - let published: ActiveRuntimeMarker = - serde_json::from_slice(&fs::read(&marker_path)?)?; - assert_eq!(published.runtime_key, "next"); - assert_eq!(leftover_marker_temp_files(&paths)?, Vec::::new()); - let _ = fs::remove_dir_all(root); - return Ok(()); - } - // Missed: the names ran out before the write ran. One name covers - // one millisecond, so when creating them is slower than one per - // millisecond no window can be wide enough; otherwise size the - // next one from what this pre-creation and write took. - let next_ms = 4 * (finished - start); - if setup_ms * 2 >= window_ms || next_ms > MAX_WINDOW_MS { - eprintln!( - "inconclusive: pre-creating {window_ms} names took {setup_ms} ms and the \ - write finished after them; this filesystem is too slow for the test \ - to say anything, so it is skipped" - ); - let _ = fs::remove_dir_all(root); - return Ok(()); - } - window_ms = window_ms.max(next_ms); - eprintln!( - "attempt {attempt}/{ATTEMPTS}: the write finished after the pre-created \ - names ran out; retrying with a {window_ms} ms window" - ); - } - eprintln!( - "inconclusive: on every attempt the write ran after the window of pre-created \ - names, so this host is too slow for this test to say anything" - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - /// Two activations writing the marker at once must both succeed, and a /// reader must find a complete marker throughout. #605 reported a writer /// that removed `active.json` before renaming the replacement in, and two From 77d8ad9c60f4074971764749f3bdf765137569c0 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Mon, 9 Nov 2026 12:04:34 +0200 Subject: [PATCH 17/17] test(runtimes): gate the marker test helpers to Unix Dropping the millis-only temp-name test left the concurrent marker test, which is Unix-only, as the sole user of test_active_runtime_marker and leftover_marker_temp_files. The Windows clippy job failed on both as dead code. Signed-off-by: Roman Inflianskas --- apps/rocm/src/runtimes_cmd.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/rocm/src/runtimes_cmd.rs b/apps/rocm/src/runtimes_cmd.rs index 74cadde934..731c1afee1 100644 --- a/apps/rocm/src/runtimes_cmd.rs +++ b/apps/rocm/src/runtimes_cmd.rs @@ -1233,6 +1233,7 @@ pub(crate) mod tests { Ok(()) } + #[cfg(unix)] fn test_active_runtime_marker(paths: &AppPaths, runtime_key: &str) -> ActiveRuntimeMarker { ActiveRuntimeMarker { runtime_id: "therock-release:gfx120X-all".to_owned(), @@ -1245,6 +1246,7 @@ pub(crate) mod tests { } } + #[cfg(unix)] fn leftover_marker_temp_files(paths: &AppPaths) -> Result> { let dir = active_runtime_marker_path(paths) .parent()