Skip to content
Merged
21 changes: 14 additions & 7 deletions apps/rocm/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4564,13 +4564,20 @@ fn apply_service_prune_plan(
/// run — the same kind of disagreement the paragraph above declines to ship.
///
/// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so
/// a second acquire on a path already held by this process would hang forever.
/// Nothing under either phase acquires this path: the only other
/// `FileLock::acquire` calls in the tree are `ensure_background_helper_running_quiet`
/// and the managed-uv bootstrap in `rocm_core::uv`, each on a different lock file,
/// and neither is reachable from here. In the other
/// direction `serve` holds this lock but never runs prune, in-process or as a
/// subprocess.
/// a second acquire on this same lock path while this process already holds
/// it would hang forever. Nothing reachable from either phase re-acquires
/// `managed_launch_lock_path`: the only other production call site on this
/// same path is `select_gpu_indices_under_launch_lock`, which is only ever
/// reached from `serve`, and `serve` never runs prune in-process or as a
/// subprocess (test code acquires this path directly too, but that is test
/// setup, not something reachable from this function's own call graph).
///
/// Separately, prune does cross a *different* lock — recording its audit
/// event takes that store's own `jsonl_lock_path` — but that call
/// (`record_cli_audit_event` in the `services` dispatch) happens only after
/// *this* function has already returned and `_launch_lock` has dropped, so
/// the two locks are never held at once and their acquisition order cannot
/// matter.
fn prune_managed_service_records(
paths: &AppPaths,
hours: u64,
Expand Down
1,710 changes: 1,663 additions & 47 deletions crates/rocm-core/src/automation.rs

Large diffs are not rendered by default.

38 changes: 2 additions & 36 deletions crates/rocm-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1143,45 +1143,11 @@ last_installed_runtime_id = "therock-release"
Ok(())
}

/// The volume serial number paired with the 64-bit file index: this
/// platform's file identity, the way an inode is on Unix.
///
/// Read through `GetFileInformationByHandle` rather than
/// `std::os::windows::fs::MetadataExt`, whose `volume_serial_number` and
/// `file_index` accessors are unstable behind the `windows_by_handle`
/// feature gate and so cannot be called on the pinned stable toolchain.
#[cfg(windows)]
#[allow(unsafe_code)] // Win32 FFI
fn windows_file_identity(path: &Path) -> Result<(u32, u64)> {
use anyhow::bail;
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::HANDLE;
use windows_sys::Win32::Storage::FileSystem::{
BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle,
};

let file =
fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?;
// All-POD `repr(C)` record, fully overwritten by the call below on
// success and never read on failure.
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
let handle = file.as_raw_handle() as HANDLE;
if unsafe { GetFileInformationByHandle(handle, &raw mut info) } == 0 {
bail!(
"failed to read the file identity of {}: {}",
path.display(),
std::io::Error::last_os_error()
);
}
Ok((
info.dwVolumeSerialNumber,
(u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow),
))
}

#[cfg(windows)]
#[test]
fn saving_the_config_replaces_the_file_rather_than_rewriting_it_in_place() -> Result<()> {
use crate::test_support::windows_file_identity;

// The Windows counterpart of the Unix test above. The file index
// together with the volume serial number is this platform's file
// identity, the way an inode is on Unix: a `fs::write` over the live
Expand Down
14 changes: 14 additions & 0 deletions crates/rocm-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,20 @@ pub fn unix_time_millis() -> u128 {
/// temp file and losing one of the two updates.
///
/// Not guaranteed:
/// - Serializing writers: this takes no lock, so two read-modify-write
/// cycles that overlap still lose one update (last rename wins). A caller
/// whose new content depends on the old must hold a lock across the whole
/// cycle, as the JSONL proposal store does (`automation::jsonl_lock_path`).
/// Two callers in the *same process* within the same millisecond are a
/// sharper case of this, not just a milder one: they generate the exact
/// same temp path below, so concurrent `fs::write`s truncating that one
/// path can publish mixed bytes, and the losing rename then fails outright
/// (its source file already gone) rather than merely losing an update —
/// this is the same-process collision
/// `automation::tests::concurrent_appends_from_many_threads_lose_no_records`'s
/// own comment describes ("two threads of one process even shared a
/// temp-file name") — fixed for its own callers by the per-store lock
/// `automation.rs` takes before ever calling into this function.
/// - Durability: the bytes are not fsynced before the rename, so a power
/// loss can still surface the new name with unflushed contents.
/// - Replacing a destination another process holds open without delete
Expand Down
26 changes: 14 additions & 12 deletions crates/rocm-core/src/process.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@
//!
//! Hidden-console/detached spawn and exit-wait on Windows, process/process-tree
//! termination on both platforms, and the exclusive `FileLock` used to
//! serialize the daemon autostart path, the managed-launch GPU claim, and
//! `uv` installs across processes.
//! serialize check-then-act sequences over shared on-disk state across
//! processes — e.g. the daemon autostart path, the managed-launch GPU claim,
//! a `uv` install, and each JSONL store's append/rewrite.
//!
//! [`run_with_timeout`] is the bulk of the file: run a command to completion
//! with both streams captured and a real time bound. A thread per pipe drains
Expand Down Expand Up @@ -387,9 +388,11 @@ pub fn process_is_running(pid: u32) -> bool {
/// Wraps the standard-library file lock (`std::fs::File::lock`), so the exclusion
/// holds between *separate `rocm` processes*, not just threads: each caller opens
/// the same lock-file path and only one can hold the lock at a time. It exists to
/// serialize check-then-act sequences over shared on-disk state — the daemon
/// autostart decision and the managed-serve GPU select-then-claim — so two
/// concurrent invocations cannot both pass the same TOCTOU check.
/// serialize check-then-act sequences over shared on-disk state — e.g. the
/// daemon autostart decision, the managed-serve GPU select-then-claim, a
/// `uv` install, and each JSONL store's append-or-repair and read-modify-write
/// (see `jsonl_lock_path` in `automation.rs`) — so two concurrent invocations
/// cannot both pass the same check, or interleave a write with another's.
///
/// The lock is released when the guard is dropped, and by the OS if the process
/// exits while holding it (so a crashed holder never wedges the next caller).
Expand Down Expand Up @@ -637,10 +640,9 @@ const DRAIN_GRACE: Duration = Duration::from_millis(200);
/// The bound matters because the message does not stop at a terminal. `rocmd`
/// renders the whole error chain into the bridge snapshot's `note`, and the
/// GPU-metrics watcher copies that note into the automation-event and audit
/// JSONL files once a minute — files whose append helpers rewrite the whole
/// file each time. Unbounded, a child that prints a megabyte and then stalls
/// past its timeout grows both logs by that much per minute, and every
/// subsequent append costs more than the last.
/// JSONL files once a minute — stores that are never rotated or capped.
/// Unbounded, a child that prints a megabyte and then stalls past its
/// timeout grows both logs by that much per minute, forever.
///
/// Sized to still hold a real stderr backtrace: the point is to stop a
/// payload-sized quote, not to shorten diagnostics.
Expand Down Expand Up @@ -1866,9 +1868,9 @@ mod tests {
//
// Unbounded, this message travels: `rocmd` renders the error chain
// into the bridge snapshot's `note`, and the GPU-metrics watcher
// writes that note into two JSONL logs every 60s, rewriting each
// file whole. A real amd-smi on a multi-GPU host prints well past
// this fixture's size.
// writes that note into two JSONL logs every 60s, neither ever
// rotated or capped. A real amd-smi on a multi-GPU host prints
// well past this fixture's size.
let mut command = Command::new(shell);
command.args([
"-c",
Expand Down
38 changes: 38 additions & 0 deletions crates/rocm-core/src/test_support.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,44 @@ use crate::{AppPaths, ManagedServiceRecord, unix_time_millis};
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};

/// The volume serial number paired with the 64-bit file index: this
/// platform's file identity, the way an inode is on Unix. Shared by every
/// test that tells a publish-by-rename apart from a rewrite in place.
///
/// Read through `GetFileInformationByHandle` rather than
/// `std::os::windows::fs::MetadataExt`, whose `volume_serial_number` and
/// `file_index` accessors are unstable behind the `windows_by_handle`
/// feature gate and so cannot be called on the pinned stable toolchain.
#[cfg(windows)]
#[allow(unsafe_code)] // Win32 FFI
pub(crate) fn windows_file_identity(path: &std::path::Path) -> anyhow::Result<(u32, u64)> {
use anyhow::{Context, bail};
use std::fs;
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::HANDLE;
use windows_sys::Win32::Storage::FileSystem::{
BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle,
};

let file =
fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?;
// All-POD `repr(C)` record, fully overwritten by the call below on
// success and never read on failure.
let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
let handle = file.as_raw_handle() as HANDLE;
if unsafe { GetFileInformationByHandle(handle, &raw mut info) } == 0 {
bail!(
"failed to read the file identity of {}: {}",
path.display(),
std::io::Error::last_os_error()
);
}
Ok((
info.dwVolumeSerialNumber,
(u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow),
))
}

pub(crate) fn temp_app_paths(name: &str) -> (PathBuf, AppPaths) {
let root = workspace_test_artifact_dir().join(format!("rocm-core-{name}-{}", unique_suffix()));
let paths = AppPaths {
Expand Down
37 changes: 33 additions & 4 deletions tests/e2e-cucumber/features/automations.feature
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ Feature: Automation watchers
# automation watcher in the CLI config and prints a confirmation; bare `rocm
# automations` lists the most recent review requests (any status — only the
# per-entry approve/reject controls are gated on "pending"). This feature
# pins the enable/disable/mode-confirmation slice plus the proposal-history
# listing's unit formatting; the watcher/daemon trigger behaviour itself is
# covered elsewhere. Config-only (no GPU, no network), so it runs on the
# mock lane every PR.
# pins the enable/disable/mode-confirmation slice, the proposal-history
# listing's unit formatting, and the bare listing's tolerance of a torn
# audit-log tail; the watcher/daemon trigger behaviour itself is covered
# elsewhere. Config-only (no GPU, no network), so it runs on the mock lane
# every PR.

@id:automations-enable-confirms-mode
Scenario: automations-01 - Enabling a watcher confirms its mode
Expand Down Expand Up @@ -54,3 +55,31 @@ Feature: Automation watchers
Given a pending artifact-download proposal approved for a size just under 1 MiB
When the user lists automations
Then the CLI reports the approved size in binary units

# Readers of the audit/events/proposals stores take no lock, so one can
# catch a write mid-append; the cut-off last line that leaves is skipped
# instead of failing the whole read.
# `render_automations_text` already swallows any read error from these
# stores via `.unwrap_or_default()`, so `rocm automations` never exits
# non-zero either way — what this scenario actually pins is that the clean
# record ahead of the torn one still shows up, rather than the torn line's
# read error wiping out the whole audit section the way it would without
# this skip.
# `render_automations_text` is the production reader of the audit store
# specifically (the one this scenario seeds). The two places a person
# reads its raw text directly are the `rocm automations` CLI subcommand
# itself, and the TUI's automations-manager screen, which shells out to
# `rocm automations list` as a subprocess — so that's where the audit
# store's tolerance is directly user-observable. It's also reachable
# through other paths that don't show the raw text the same way —
# `rocm chat`'s and the TUI's slash-command `rocm_command` tool in-process
# (the model sees it, or the TUI shrinks it to field counts first), and
# rocmd's own MCP `rocm_command` tool via a further `rocm automations list`
# subprocess — this isn't meant as an exhaustive map of every caller, just
# context for why the CLI subcommand is the right level to pin this at.
@id:automations-tolerates-torn-audit-tail
Scenario: automations-06 - Listing automations tolerates a torn audit-log tail
Given a fresh CLI configuration
And the audit event log ends in a torn, unterminated line
When the user lists automations
Then the CLI still lists the record before the torn tail
9 changes: 4 additions & 5 deletions tests/e2e-cucumber/features/examine.feature
Original file line number Diff line number Diff line change
Expand Up @@ -367,11 +367,10 @@ Feature: GPU detection and system inspection
# With an amd-smi that prints a snapshot's worth of JSON and only then
# stalls, that quote is the entire payload, and the note is not a terminal
# message: `record_gpu_metrics` copies it into the automation-event and audit
# JSONL files every 60s, and both append helpers rewrite the whole file each
# time. Unbounded, the logs grew by the payload's size every minute and each
# write cost more than the last. So the note must name the timeout, keep
# enough of the child's output to be worth reading, say that it was shortened
# -- and stay small.
# JSONL files every 60s, neither ever rotated or capped. Unbounded, the logs
# grew by the payload's size every minute, forever. So the note must name
# the timeout, keep enough of the child's output to be worth reading, say
# that it was shortened -- and stay small.
#
# Asserted here rather than only as a unit test because the note's journey is
# the point: the bound lives in `rocm-core`'s `run_with_timeout` and the
Expand Down
59 changes: 55 additions & 4 deletions tests/e2e-cucumber/tests/e2e/automations_steps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,13 @@
//! first plants an automation runtime-state marking the daemon already
//! running under THIS test process's (live) pid, so the CLI's double-spawn
//! guard skips the spawn. Contracts verified against the running Linux binary
//! (EAI-8072). Scoped to the enable/disable/mode slice plus the
//! proposal-history listing's unit formatting; the watcher/daemon trigger
//! behaviour itself is covered separately.
//! (EAI-8072). Scoped to the enable/disable/mode slice, the proposal-history
//! listing's unit formatting, and the bare listing's tolerance of a torn
//! audit-log tail; the watcher/daemon trigger behaviour itself is covered
//! separately.

use cucumber::{given, then, when};
use rocm_core::{AutomationProposalRecord, append_automation_proposal};
use rocm_core::{AuditEventRecord, AutomationProposalRecord, append_automation_proposal};

use crate::E2eWorld;

Expand Down Expand Up @@ -108,6 +109,39 @@ async fn pending_artifact_download_proposal(world: &mut E2eWorld) {
.expect("failed to write automation proposal fixture");
}

/// Seed the audit-event store with one clean record followed by a cut-off
/// JSON fragment with no trailing newline — the shape a writer killed
/// mid-append leaves. See the scenario's own comment in `automations.feature`
/// for why this matters and who reads this store; see
/// `scan_recent_jsonl_records`/`is_torn_record` in `rocm-core` for the read
/// path this exercises (not `inspect_tail`, which is the write path's own,
/// separate torn-tail check).
#[given("the audit event log ends in a torn, unterminated line")]
async fn torn_audit_tail(world: &mut E2eWorld) {
let paths = world.app_paths();
paths.ensure().expect("failed to create app dirs");
let clean = AuditEventRecord {
at_unix_ms: 1_700_000_000_000,
source: "e2e".to_owned(),
category: "concurrency".to_owned(),
actor: "writer".to_owned(),
level: "info".to_owned(),
action: "append".to_owned(),
message: "clean".to_owned(),
watcher_id: None,
service_id: None,
};
let clean_line = serde_json::to_string(&clean).expect("serialize audit record");
// Just the first 20 bytes of the same record: short enough that the cut
// falls well before the `message` field, so what it says doesn't matter.
let torn_fragment = &clean_line[..20];
std::fs::write(
paths.audit_events_path(),
format!("{clean_line}\n{torn_fragment}"),
)
.expect("failed to seed a torn audit-event tail");
}

#[given("an enabled automation watcher")]
async fn enabled_watcher(world: &mut E2eWorld) {
suppress_daemon_spawn(world);
Expand Down Expand Up @@ -166,6 +200,23 @@ async fn enable_unknown(world: &mut E2eWorld) {

// ── Then ───────────────────────────────────────────────────────────

#[then("the CLI still lists the record before the torn tail")]
async fn still_lists_record_before_torn_tail(world: &mut E2eWorld) {
// The count, not rc == 0, is the real assertion here (see
// `automations.feature` for why). `audit_event_plain_summary` renders
// every audit event's category to one of a handful of fixed strings
// (never the raw message), so the fixture's "concurrency" category falls
// through to the generic one; exactly one occurrence confirms the one
// clean record before the torn tail was actually read and rendered, not
// silently dropped along with it (zero) or duplicated (more than one).
let out = ok_output(world);
let activity_lines = out.matches("Background activity was recorded.").count();
assert_eq!(
activity_lines, 1,
"expected exactly the one valid record before the torn tail to be listed, got:\n{out}"
);
}

#[then(regex = r"^the CLI confirms the watcher is enabled in (observe|propose) mode$")]
async fn confirm_enabled(world: &mut E2eWorld, mode: String) {
let out = ok_output(world);
Expand Down
Loading