From c6050117c1c4d7be9c29559e211fc9a2e77dcb8e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 08:17:58 +0000 Subject: [PATCH 1/5] fix(core): fall back to truncating a torn JSONL tail when the atomic replace fails Repairing a torn tail rewrites the whole store and renames it into place. When that rewrite cannot complete (a full disk, or on Windows another program holding the store open without delete sharing), the repair had no fallback: the debris stayed, and every later append to that store hit the same failure, which also kept the daemon from starting. Only when the replace fails, cut the debris in place under the store lock and do a plain append. A truncate needs no free space and no rename, so it can succeed where the replace cannot. The normal repair path is unchanged, the fallback prints nothing, and if both steps fail the error names the store and both causes. Fixes #748 Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 380 +++++++++++++++++++++++++---- 1 file changed, 336 insertions(+), 44 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index f1a286ebd6..5bfebbe2d1 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -19,7 +19,10 @@ //! its own new record as a single rewrite via `write_atomic`, so a lock-free //! reader never observes a file shrunk mid-repair. That rewrite costs O(store) //! rather than O(record), the same as a proposal status update, which rewrites -//! the whole store via `write_atomic` too. Nothing in this module ever +//! the whole store via `write_atomic` too. Only when that rewrite itself fails +//! (a full disk, or on Windows another program holding the store open without +//! delete sharing) does the append cut the debris in place instead, so a store +//! cannot be wedged by a failed repair; otherwise nothing in this module ever //! shrinks a store in place. Readers take no lock, and an unterminated last //! line that is not valid JSON — an append caught mid-write — is skipped //! rather than reported as corruption. @@ -244,7 +247,8 @@ pub struct AuditEventRecord { /// rewrite needs it for its read-modify-write; an append needs it both to /// serialize against other appends and because it can itself replace or /// rename the live file out from under a concurrent writer — a torn tail is -/// repaired by renaming a new file over the store (see [`TailState::Torn`]), +/// repaired by renaming a new file over the store, or by truncating it when +/// that fails (see [`TailState::Torn`]), /// and a rotating events or audit store is renamed aside to `.1` (see /// [`rotate_jsonl_generations`]) — and an append that had already opened the /// old file would otherwise land in the replaced or rotated-away copy and @@ -367,8 +371,9 @@ fn open_jsonl_for_append(path: &Path) -> Result { /// back, and the debris itself is discarded by the *next* append rather than /// being cut first as a separate step — see [`inspect_tail`] and /// [`TailState::Torn`]. A reader already skips the debris in the meantime. -/// Nothing in this function ever truncates the live file, so a lock-free -/// reader never observes it shrink. +/// This function truncates the live file only as the last resort when that +/// replacement itself fails (see [`repair_torn_tail_and_append`]), so a +/// lock-free reader can observe it shrink only then, never on the normal path. /// /// [`append_or_tolerate_a_missing_newline`] checks whether a failed write /// landed anyway before reporting it, so a write that fails but still lands @@ -419,31 +424,14 @@ fn append_jsonl_record( } } match inspect_tail(path, &file)? { - TailState::Torn(tail_start) => { - // A torn tail is debris — crash or a failed write — that must be - // cut before this record lands. Cutting it in place and then - // appending as a second step would briefly shrink the file for - // any lock-free reader mid-scan, and regrowing it afterward can - // make that scan splice bytes read at two different moments into - // something that parses as a record nobody wrote (the hazard - // `ReverseLines::next_line`'s short-read guard exists for). - // Replacing the whole file atomically instead means a reader - // only ever sees the pre-repair length or this append's, never a - // length in between. - let mut content = read_range(path, &file, 0, tail_start)?; - content.extend_from_slice(&line); - // Done with the pre-repair file's handle before replacing it, - // rather than leaving that to whatever happens to still have - // `file` open at function return. The rename already succeeds - // with this handle open regardless — std opens files with - // `FILE_SHARE_DELETE` by default on Windows, so a rename isn't - // blocked by it (see `write_atomic`'s doc for the case that - // *isn't* covered: another process holding the destination open - // without delete-sharing) — but this states the intent rather - // than leaning on that default. - drop(file); - crate::write_atomic(path, &content, "jsonl")?; - } + TailState::Torn(tail_start) => repair_torn_tail_and_append( + path, + file, + tail_start, + &line, + |path, content| crate::write_atomic(path, content, "jsonl"), + truncate_jsonl_in_place, + )?, TailState::ValidUnterminated(start) => { line.insert(0, b'\n'); append_or_tolerate_a_missing_newline(path, &file, &line, start)?; @@ -455,6 +443,76 @@ fn append_jsonl_record( Ok(()) } +/// Discard the torn tail of the store at `path`, which starts at `tail_start`, +/// and land `line` in its place. The caller must hold the store's +/// [`jsonl_lock_path`] lock; `file` is its handle on the pre-repair store. +/// +/// The repair publishes the clean prefix plus `line` as one `replace`, so a +/// lock-free reader only ever sees the pre-repair length or the post-append +/// length (see [`append_jsonl_record`]). That rewrite can itself fail — no +/// room for the copy on a full disk, or on Windows another program holding +/// the store open without delete sharing — and leaving it there would wedge +/// the store: every later append would hit the same debris and the same +/// failure. Only then does this fall back to `truncate`, cutting the debris +/// in place and appending `line` as a plain append. That reopens, on this +/// already-failing path only, the shrink a lock-free reader can briefly +/// observe; a truncate frees space rather than needing it and renames +/// nothing, so it can succeed where `replace` cannot. Like a repair by +/// `replace`, the fallback prints nothing. +/// +/// If `truncate` fails too, the error names both causes and the store is left +/// as it was; readers keep skipping the debris. +/// +/// `replace` and `truncate` are parameters so a test can make either fail on +/// demand. +fn repair_torn_tail_and_append( + path: &Path, + file: fs::File, + tail_start: u64, + line: &[u8], + replace: impl FnOnce(&Path, &[u8]) -> Result<()>, + truncate: impl FnOnce(&Path, u64) -> Result<()>, +) -> Result<()> { + let mut content = read_range(path, &file, 0, tail_start)?; + content.extend_from_slice(line); + // Done with the pre-repair file's handle before replacing it, rather than + // leaving that to whatever happens to still have `file` open at function + // return. The rename already succeeds with this handle open regardless — + // std opens files with `FILE_SHARE_DELETE` by default on Windows, so a + // rename isn't blocked by it (see `write_atomic`'s doc for the case that + // *isn't* covered: another process holding the destination open without + // delete-sharing) — but this states the intent rather than leaning on that + // default. + drop(file); + let replace_error = match replace(path, &content) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + drop(content); + if let Err(truncate_error) = truncate(path, tail_start) { + bail!( + "failed to repair the torn tail of {}: replacing it failed ({replace_error:#}); \ + truncating it failed ({truncate_error:#})", + path.display() + ); + } + let file = open_jsonl_for_append(path)?; + append_or_tolerate_a_missing_newline(path, &file, line, tail_start) +} + +/// Cut the store at `path` down to `len` bytes through a fresh write handle. +/// +/// Not the append handle: on Windows an append-only handle lacks +/// `FILE_WRITE_DATA`, which `set_len` needs. +fn truncate_jsonl_in_place(path: &Path, len: u64) -> Result<()> { + fs::OpenOptions::new() + .write(true) + .open(path) + .with_context(|| format!("failed to open {} for writing", path.display()))? + .set_len(len) + .with_context(|| format!("failed to truncate {}", path.display())) +} + /// The warning to print that rotating `path` failed, or `None` after the first /// one for that store in this process, so a rename that keeps failing (e.g. /// another program holds the live file open without delete sharing on @@ -545,8 +603,9 @@ enum TailState { ValidUnterminated(u64), /// Debris — crash, or a write that failed partway through a genuinely /// incomplete record — starting at this offset. Must be cut before the - /// next record lands, but never by truncating in place — see - /// [`append_jsonl_record`]. + /// next record lands, by replacing the store rather than truncating it in + /// place — except as the fallback when that replacement fails; see + /// [`append_jsonl_record`] and [`repair_torn_tail_and_append`]. Torn(u64), } @@ -583,12 +642,13 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// Read bytes `[start, end)` of `file`. /// /// Returns fewer bytes if the file now ends before `end`. Nothing this -/// module's own writers do can shrink a store in place — an append only ever +/// module's own writers do shrinks a store in place — an append only ever /// grows it, and a torn-tail repair replaces it wholesale via `write_atomic` -/// rather than truncating the live file (see [`inspect_tail`] and -/// [`TailState::Torn`]) — so a short read here means the store was edited or -/// truncated out-of-band, a case this module does not try to recover from -/// cleanly. Most callers treat what is still there the same as any other +/// (see [`inspect_tail`] and [`TailState::Torn`]) — except the repair's +/// fallback, taken only when that replacement fails and only under the store's +/// lock (see [`repair_torn_tail_and_append`]); otherwise a short read here +/// means the store was edited or truncated out-of-band, a case this module +/// does not try to recover from cleanly. Most callers treat what is still there the same as any other /// unexpectedly-ending tail; [`ReverseLines`] is the one exception, and /// reports an error instead once it would have to splice such a read onto /// bytes it already read, rather than risk inventing a line. @@ -775,10 +835,12 @@ fn open_jsonl_generations(path: &Path, generations: usize) -> Result( path: &Path, file: &fs::File, @@ -923,8 +985,8 @@ impl<'a> ReverseLines<'a> { // A short read while `pending` is empty — nothing accumulated // yet, whether from an earlier call or from this one — just // means the store was edited or truncated out-of-band (see - // `read_range`); nothing a concurrent writer does can shrink it - // in place. Once `pending` holds any bytes, a short read means + // `read_range`); no concurrent writer shrinks it in place short of + // the torn-tail repair's rare fallback. Once `pending` holds any bytes, a short read means // they no longer continue from it, so splicing them would // invent a line. if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { @@ -2247,8 +2309,8 @@ mod tests { Ok(()) } - // Nothing in this module's own writers shrinks a store, but an - // out-of-band edit might, and load_recent_jsonl_records is supposed to + // Nothing in this module's own writers shrinks a store (bar the torn-tail + // repair's rare fallback), but an out-of-band edit might, and load_recent_jsonl_records is supposed to // retry from the new length rather than fail. Driving that needs the // shrink to land strictly between two of ReverseLines's own chunked // reads (see the `test_hooks` module doc) — which in turn @@ -2476,6 +2538,236 @@ mod tests { Ok(()) } + /// The store's file identity, as a string so one assertion covers the + /// inode on Unix and the volume serial plus file index on Windows. + #[cfg(unix)] + fn file_identity(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + Ok(fs::metadata(path)?.ino().to_string()) + } + + #[cfg(windows)] + fn file_identity(path: &Path) -> Result { + Ok(format!( + "{:?}", + crate::test_support::windows_file_identity(path)? + )) + } + + /// Run the torn-tail repair on the seeded audit store with the replace and + /// truncate steps supplied by the test, appending the record `b`. + fn repair_audit_store_with( + paths: &AppPaths, + replace: impl FnOnce(&Path, &[u8]) -> Result<()>, + truncate: impl FnOnce(&Path, u64) -> Result<()>, + ) -> Result<()> { + let path = paths.audit_events_path(); + let file = open_jsonl_for_append(&path)?; + let TailState::Torn(tail_start) = inspect_tail(&path, &file)? else { + bail!("the seeded store has no torn tail"); + }; + let mut line = audit_line("b").into_bytes(); + line.push(b'\n'); + repair_torn_tail_and_append(&path, file, tail_start, &line, replace, truncate) + } + + // The fallback the whole-store rewrite lacked: when it cannot complete + // (a full disk, a sharing violation), the debris is cut in place instead + // of wedging every later append on it. The same file identity afterwards + // is what tells that apart from the replace the test above pins. + #[test] + fn a_torn_tail_is_cut_in_place_when_replacing_the_store_fails() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-truncates"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + let mut attempted = Vec::new(); + + repair_audit_store_with( + &paths, + |_, content| { + attempted = content.to_vec(); + bail!("no space left on device") + }, + truncate_jsonl_in_place, + )?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + let expected = format!("{}\n{}\n", audit_line("a"), audit_line("b")); + assert_eq!( + String::from_utf8(attempted)?, + expected, + "the whole-store replace must be tried first, with the full repaired store" + ); + assert_eq!(text?, expected); + assert_eq!(before, after, "the fallback must cut the debris in place"); + assert_eq!(audit_messages(&recent?), ["a", "b"]); + Ok(()) + } + + // The fallback is a last resort: a replace that works must not be + // followed by a truncate, or the shrink window it closes would be back on + // every repair. + #[test] + fn a_torn_tail_is_not_truncated_when_replacing_the_store_works() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-unused"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + + repair_audit_store_with( + &paths, + |path, content| crate::write_atomic(path, content, "jsonl"), + |_, _| bail!("truncate must not run when the replace succeeded"), + )?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_ne!(before, after, "a working replace publishes a new file"); + Ok(()) + } + + #[test] + fn a_torn_tail_that_can_be_neither_replaced_nor_cut_reports_both_causes() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-both-fail"); + let torn = &audit_line("torn")[..20]; + let seeded = format!("{}\n{torn}", audit_line("a")); + seed_audit_store(&paths, &seeded)?; + let path = paths.audit_events_path(); + + let result = repair_audit_store_with( + &paths, + |_, _| bail!("no space left on device"), + |_, _| bail!("permission denied"), + ); + + let text = fs::read_to_string(&path); + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert_eq!( + format!("{:#}", result.expect_err("both steps failed")), + format!( + "failed to repair the torn tail of {}: replacing it failed (no space left on \ + device); truncating it failed (permission denied)", + path.display() + ) + ); + assert_eq!( + text?, seeded, + "a failed repair must leave the store as it was" + ); + assert_eq!(audit_messages(&recent?), ["a"]); + Ok(()) + } + + // After the fallback the store is clean, so the next append is the plain + // one-line append: no second repair, no new file. + #[test] + fn appends_after_a_fallback_repair_are_plain_appends() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-then-append"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + repair_audit_store_with( + &paths, + |_, _| bail!("no space left on device"), + truncate_jsonl_in_place, + )?; + let repaired = file_identity(&path)?; + + append_audit_event(&paths, &audit_record("c"))?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!( + text?, + format!( + "{}\n{}\n{}\n", + audit_line("a"), + audit_line("b"), + audit_line("c") + ) + ); + assert_eq!(repaired, after); + Ok(()) + } + + // The real trigger rather than an injected one: a store directory that + // takes no new files, so the temp copy of the whole-store replace cannot + // be created, while the store file itself can still be written. + #[cfg(unix)] + #[test] + fn append_repairs_a_torn_tail_in_a_directory_that_takes_no_new_files() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let (root, paths) = temp_app_paths("torn-tail-read-only-dir"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let dir = path.parent().context("the audit store has a directory")?; + // Create the lock file now: the append reuses an existing one, but + // could not create it inside the read-only directory. + drop(FileLock::acquire(jsonl_lock_path(&path))?); + let before = file_identity(&path)?; + + fs::set_permissions(dir, fs::Permissions::from_mode(0o555))?; + let read_only_holds = fs::File::create(dir.join("probe")).is_err(); + let result = read_only_holds.then(|| append_audit_event(&paths, &audit_record("b"))); + fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + let Some(result) = result else { + // Running as a user the permission bits do not bind (root). + eprintln!("skipped: a read-only directory still takes new files for this user"); + return Ok(()); + }; + result?; + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(before, after); + Ok(()) + } + + // Windows' real trigger: another program holds the store open without + // delete sharing (a virus scanner or indexer, say), so the rename over it + // is refused while the store itself can still be written. + #[cfg(windows)] + #[test] + fn append_repairs_a_torn_tail_while_another_handle_blocks_replacing_the_store() -> Result<()> { + use std::os::windows::fs::OpenOptionsExt; + + const FILE_SHARE_READ_WRITE: u32 = 0x1 | 0x2; + let (root, paths) = temp_app_paths("torn-tail-share-violation"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + let blocker = fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ_WRITE) + .open(&path)?; + + let result = append_audit_event(&paths, &audit_record("b")); + + drop(blocker); + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + result?; + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(before, after); + Ok(()) + } + #[cfg(unix)] #[test] fn append_to_a_clean_or_unterminated_store_keeps_the_same_file() -> Result<()> { From bc501b6ffd9875432bb9bab7e22a7f8581e23183 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 09:30:00 +0000 Subject: [PATCH 2/5] fix(core): keep both causes when a torn-tail repair fails, and state the residual reader hazard Address review of the truncate fallback. The test for the two-failure error used flat stub errors, so it stayed green if either cause was formatted without its context chain; it now uses layered errors like the real replace and truncate steps. When the cut succeeds but the append after it fails, the error now also carries the replace failure, which is the root cause, and a test covers it. Docs: one comment still said a torn-tail repair always replaces the live file. The reader-side docs claimed a retry covers every shrink the fallback can cause; a debris line longer than a scan chunk that is then regrown to its old length is not detected as a shrink, so say that instead. Note on the Unix real-trigger test that it passes without testing anything when run as root. Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 106 +++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 19 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 5bfebbe2d1..7b29be370c 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -454,14 +454,24 @@ fn append_jsonl_record( /// the store open without delete sharing — and leaving it there would wedge /// the store: every later append would hit the same debris and the same /// failure. Only then does this fall back to `truncate`, cutting the debris -/// in place and appending `line` as a plain append. That reopens, on this -/// already-failing path only, the shrink a lock-free reader can briefly -/// observe; a truncate frees space rather than needing it and renames -/// nothing, so it can succeed where `replace` cannot. Like a repair by -/// `replace`, the fallback prints nothing. +/// in place and appending `line` as a plain append. A truncate frees space +/// rather than needing it and renames nothing, so it can succeed where +/// `replace` cannot. Like a repair by `replace`, the fallback prints nothing. +/// +/// The fallback reopens, on this already-failing path only, the shrink a +/// lock-free reader can observe, and the reader's guards narrow that window +/// without closing it: a scan that comes back short is refused or retried +/// (see [`ReverseLines`] and `load_recent_jsonl_records_from`), but one that +/// already read part of a debris line longer than a scan chunk and then finds +/// the file regrown to at least its old length reads on undetected, and can +/// splice the two reads into a record nobody wrote. That is the cost of not +/// wedging the store, accepted for a path that needs a failing replace, a +/// debris line that long and a reader mid-scan all at once. /// /// If `truncate` fails too, the error names both causes and the store is left -/// as it was; readers keep skipping the debris. +/// as it was; readers keep skipping the debris. If the append after a +/// successful truncate fails, the error carries the replace failure too: it is +/// the root cause. /// /// `replace` and `truncate` are parameters so a test can make either fail on /// demand. @@ -496,8 +506,15 @@ fn repair_torn_tail_and_append( path.display() ); } - let file = open_jsonl_for_append(path)?; - append_or_tolerate_a_missing_newline(path, &file, line, tail_start) + open_jsonl_for_append(path) + .and_then(|file| append_or_tolerate_a_missing_newline(path, &file, line, tail_start)) + .with_context(|| { + format!( + "failed to append to {} after cutting its torn tail \ + (replacing the store had failed: {replace_error:#})", + path.display() + ) + }) } /// Cut the store at `path` down to `len` bytes through a fresh write handle. @@ -753,8 +770,8 @@ fn load_recent_jsonl_records( /// A rotation can rename files between two of the opens: opening the live /// file and then `.1` across one would open the same file twice (and read its /// records twice), and opening `.1` and then the live file across one would -/// skip a generation. A torn-tail repair (see [`TailState::Torn`]) also -/// replaces the live file, by renaming a new one over it — not a rotation, +/// skip a generation. A torn-tail repair (see [`TailState::Torn`]) normally +/// also replaces the live file, by renaming a new one over it — not a rotation, /// but the same hazard for this snapshot: a path opened before the repair /// would still read the pre-repair file by handle, yet a path opened after /// would see different content at the same name. So after opening them all, @@ -839,8 +856,9 @@ fn open_jsonl_generations(path: &Path, generations: usize) -> Result( path: &Path, file: &fs::File, @@ -986,9 +1004,9 @@ impl<'a> ReverseLines<'a> { // yet, whether from an earlier call or from this one — just // means the store was edited or truncated out-of-band (see // `read_range`); no concurrent writer shrinks it in place short of - // the torn-tail repair's rare fallback. Once `pending` holds any bytes, a short read means - // they no longer continue from it, so splicing them would - // invent a line. + // the torn-tail repair's rare fallback. Once `pending` holds any + // bytes, a short read means they no longer continue from it, so + // splicing them would invent a line. if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { bail!("{} shrank while it was being read", self.path.display()); } @@ -2642,10 +2660,18 @@ mod tests { seed_audit_store(&paths, &seeded)?; let path = paths.audit_events_path(); + // Layered like the real steps (`write_atomic` and + // `truncate_jsonl_in_place` both wrap the OS error in a context), so + // the assertion fails if either cause is formatted without its chain. let result = repair_audit_store_with( &paths, - |_, _| bail!("no space left on device"), - |_, _| bail!("permission denied"), + |_, _| { + Err(anyhow::anyhow!("no space left on device")) + .context("failed to replace the store") + }, + |_, _| { + Err(anyhow::anyhow!("permission denied")).context("failed to truncate the store") + }, ); let text = fs::read_to_string(&path); @@ -2654,8 +2680,9 @@ mod tests { assert_eq!( format!("{:#}", result.expect_err("both steps failed")), format!( - "failed to repair the torn tail of {}: replacing it failed (no space left on \ - device); truncating it failed (permission denied)", + "failed to repair the torn tail of {}: replacing it failed (failed to replace \ + the store: no space left on device); truncating it failed (failed to truncate \ + the store: permission denied)", path.display() ) ); @@ -2667,6 +2694,42 @@ mod tests { Ok(()) } + // The replace failure is the root cause, so it must survive into the error + // when the cut itself works but the append after it does not. The truncate + // stub swaps the store for a directory, which makes reopening it fail. + #[test] + fn a_failed_append_after_the_cut_still_reports_why_the_replace_failed() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-append-fails"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + + let result = repair_audit_store_with( + &paths, + |_, _| { + Err(anyhow::anyhow!("no space left on device")) + .context("failed to replace the store") + }, + |path, _| { + fs::remove_file(path)?; + fs::create_dir(path)?; + Ok(()) + }, + ); + + fs::remove_dir_all(root).ok(); + let message = format!("{:#}", result.expect_err("the append after the cut failed")); + assert!( + message.starts_with(&format!( + "failed to append to {} after cutting its torn tail (replacing the store had \ + failed: failed to replace the store: no space left on device): ", + path.display() + )), + "unexpected error: {message}" + ); + Ok(()) + } + // After the fallback the store is clean, so the next append is the plain // one-line append: no second repair, no new file. #[test] @@ -2703,6 +2766,11 @@ mod tests { // The real trigger rather than an injected one: a store directory that // takes no new files, so the temp copy of the whole-store replace cannot // be created, while the store file itself can still be written. + // + // This is the only test that drives the production `replace`/`truncate` + // wiring in `append_jsonl_record` rather than injected steps, and it passes + // without testing anything for a user the permission bits do not bind + // (root). CI runs it as a regular user. #[cfg(unix)] #[test] fn append_repairs_a_torn_tail_in_a_directory_that_takes_no_new_files() -> Result<()> { From fbef4b2e0b9ac23d07ef375c993b3f47fdc3dc48 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:37:07 +0000 Subject: [PATCH 3/5] test(e2e): cover the torn audit-log repair fallback through an audited command Add automations-10: with a torn audit log and an audit directory that takes no new files, `rocm config set-permissions` must still record its event, print no audit-log warning, keep the earlier record and drop the torn line. Without the fallback it reproduces the reported symptom (a warning on every audited command and a lost event). A directory permission does not bind root, and the self-hosted WSL2 guest runs the suite as root, so the scenario is gated by a new `@requires-dir-permissions` tag backed by a capability probe, following `@requires-case-sensitive-fs`, rather than passing there without testing anything. The audit directory is made writable again in the world's teardown so the temp root can be removed. Signed-off-by: Jussi Elo --- docs/testing.md | 13 ++- tests/e2e-cucumber/README.md | 1 + .../e2e-cucumber/features/automations.feature | 26 +++++- tests/e2e-cucumber/src/capability.rs | 86 +++++++++++++++++++ tests/e2e-cucumber/src/expectation.rs | 74 ++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 6 ++ .../tests/e2e/automations_steps.rs | 86 ++++++++++++++++++- 7 files changed, 288 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 782e892e75..d791a2cd4d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1500,9 +1500,18 @@ rotation policy; two further tests go through the public `append_automation_event`/`append_audit_event` and `load_recent_automation_events`/`load_recent_audit_events` wrappers instead, at the real size cap, so the wiring between them and the rotation policy -itself stays covered. The user-visible side is `automations-07` through -`automations-09` in +itself stays covered. The repair of a torn tail, which normally replaces the +store and falls back to cutting the debris in place when that replacement +fails, has its own tests there: each step injected to fail, the error naming +both causes when both fail, and a real fault on each platform (a directory +that takes no new files on Unix, a handle without delete sharing on Windows). +The user-visible side is `automations-07` through `automations-09`, plus +`automations-10` for the fallback, in [automations.feature](../tests/e2e-cucumber/features/automations.feature). +`automations-10` makes the audit directory read-only, which does not bind root, +so it carries `@requires-dir-permissions` and resolves to a skip for a user the +permission bits do not bind (the self-hosted WSL2 guest runs as root) rather +than pass without testing the fallback. Natural-language plan handoff tests: diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index a9852966e3..0834c45eb3 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -141,6 +141,7 @@ Scenarios carry stable-id and capability tags: | `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. | | `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | | `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | +| `@requires-dir-permissions` | Premise is a directory that refuses new files once its write bits are cleared. Resolves to **skip** for a user those bits do not bind: root, which is what the self-hosted WSL2 guest runs the suite as. The capability probe's `dir_permissions_bind` clears the write bits on a directory under the temp root and tries to create a file in it. `@requires-os:linux` cannot express this: root on Linux is still `linux`, and the scenario would pass without testing anything. Always **skip** on Windows, which has no such bits. | | `@requires-case-sensitive-fs` | Premise is two files whose names differ only in letter case. Resolves to **skip** where the scenarios' temp root folds case. The capability probe's `case_sensitive_fs` writes two such names in its own temp dir, which follows `$TMPDIR` like every scenario's isolated root, and checks they stayed two files. `@requires-os:linux` cannot express this: `$TMPDIR` on Linux, WSL2 included, can sit on a casefolded directory or a mounted Windows drive. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index 372846979c..fc08c39d1c 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -6,7 +6,8 @@ Feature: Automation watchers # per-entry approve/reject controls are gated on "pending"). This feature # pins the enable/disable/mode-confirmation slice, the proposal-history # listing's unit formatting, the bare listing's tolerance of a torn - # audit-log tail, and how the audit log and proposal history are kept + # audit-log tail, how the next write repairs one, and how the audit log + # and proposal history are kept # bounded; the watcher/daemon trigger behaviour itself is covered # elsewhere. Config-only (no GPU, no network), so it runs on the mock lane # every PR. @@ -127,3 +128,26 @@ Feature: Automation watchers When the user selects a permissions mode Then each rotated generation holds what the previous one held And the oldest previous generation is gone + + # A write that was cut off leaves a torn last line, and the next append + # repairs it by writing a clean copy of the audit log beside it and renaming + # that over the original. When the copy cannot be made — a full disk, or on + # Windows another program holding the log open without delete sharing — the + # repair used to give up: the debris stayed, the CLI warned that it failed to + # write the audit event on every command that records one, and the event was + # lost. The repair now falls back to cutting the debris in place. A directory + # that takes no new files is the same failure without filling a disk, so this + # drives it through `rocm config set-permissions`, a command the CLI audits. + # The fault is a directory permission, which does not bind root (the + # self-hosted WSL2 guest runs the suite as root), so `@requires-dir-permissions` + # skips it there rather than let it pass without testing anything. The unit + # tests in `automation.rs` cover the other triggers (an injected failure of + # each step, and on Windows a handle without delete sharing). + @id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced @requires-os:linux @requires-dir-permissions + Scenario: automations-10 - A command still records its audit event when the torn audit log cannot be replaced + Given the audit event log ends in a torn, unterminated line + And the audit log's directory takes no new files + When the user selects a permissions mode + Then the CLI confirms the permissions mode + And the CLI prints no warning about the audit log + And the audit log keeps its earlier record, drops the torn line and records the permissions change diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 3f4a037276..244b4d04c6 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -151,6 +151,10 @@ pub struct HostCapability { /// the temp root the scenarios' isolated roots are created in. Gates /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`]. pub case_sensitive_fs: bool, + /// Whether a directory made read-only refuses new files to the user + /// running the suite. Gates `@requires-dir-permissions`; see + /// [`probe_dir_permissions_bind`]. + pub dir_permissions_bind: bool, } impl HostCapability { @@ -411,6 +415,7 @@ fn probe_host_capability() -> HostCapability { let platform_slug = derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family, is_wsl); let case_sensitive_fs = probe_case_sensitive_fs(root); + let dir_permissions_bind = probe_dir_permissions_bind(root); HostCapability { os_family, @@ -422,9 +427,44 @@ fn probe_host_capability() -> HostCapability { effective_serve_engine, platform_slug, case_sensitive_fs, + dir_permissions_bind, } } +/// Whether a directory with its write bits cleared refuses to create a file for +/// the user running the suite. +/// +/// Probed rather than inferred from the OS: root is not bound by directory +/// permissions, and the self-hosted WSL2 guest runs the suite as root even +/// though it is `os_family` `linux`. A scenario whose fault is "this directory +/// takes no new files" would then pass without exercising anything, so it is +/// skipped where this answers `false`. +/// +/// The probe's directory is created under `dir`, the temp root the scenarios' +/// isolated roots follow, so it answers for the filesystem they write to, and +/// its permissions are restored before it returns so the temp root can be +/// removed. Any I/O error answers `false`: the premise has not been shown to +/// hold. Always `false` where there are no Unix permission bits to clear. +#[cfg(unix)] +fn probe_dir_permissions_bind(dir: &std::path::Path) -> bool { + use std::os::unix::fs::PermissionsExt; + + let probe = || -> std::io::Result { + let probe_dir = dir.join("permission-probe"); + std::fs::create_dir_all(&probe_dir)?; + std::fs::set_permissions(&probe_dir, std::fs::Permissions::from_mode(0o555))?; + let created = std::fs::File::create(probe_dir.join("probe-file")); + std::fs::set_permissions(&probe_dir, std::fs::Permissions::from_mode(0o755))?; + Ok(created.is_err()) + }; + probe().unwrap_or(false) +} + +#[cfg(not(unix))] +fn probe_dir_permissions_bind(_dir: &std::path::Path) -> bool { + false +} + /// Whether `dir`'s filesystem keeps two names that differ only in letter case /// apart. /// @@ -836,6 +876,7 @@ mod tests { effective_serve_engine: "lemonade".to_owned(), platform_slug: "strix-halo".to_owned(), case_sensitive_fs: false, + dir_permissions_bind: true, }; assert!(strix.engine_available("lemonade")); // vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu. @@ -851,6 +892,7 @@ mod tests { effective_serve_engine: "vllm".to_owned(), platform_slug: "mi300x".to_owned(), case_sensitive_fs: true, + dir_permissions_bind: true, }; assert!(mi300x.engine_available("vllm")); assert!(mi300x.engine_available("lemonade")); @@ -1168,4 +1210,48 @@ Local model engines std::fs::write(¬_a_dir, "").expect("write file"); assert!(!probe_case_sensitive_fs(¬_a_dir)); } + + /// A probe that cannot even make its directory has not shown the premise + /// holds, so the gate must skip rather than let the scenario fail. + #[test] + fn permission_probe_answers_false_when_it_cannot_write() { + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let not_a_dir = tmp.path().join("plain-file"); + std::fs::write(¬_a_dir, "").expect("write file"); + assert!(!probe_dir_permissions_bind(¬_a_dir)); + } + + /// The probe's directory is read-only while it probes; it must be given + /// back, or the temp root it lives in could not be removed. + #[cfg(unix)] + #[test] + fn permission_probe_leaves_its_directory_removable() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + probe_dir_permissions_bind(tmp.path()); + let mode = std::fs::metadata(tmp.path().join("permission-probe")) + .expect("probe dir") + .permissions() + .mode(); + assert_eq!(mode & 0o200, 0o200, "owner write bit was not restored"); + } + + /// The answer the gated scenario relies on: where the suite runs as an + /// ordinary user, a read-only directory refuses new files. Skipped as root, + /// for whom the probe is rightly `false`. + #[cfg(unix)] + #[test] + fn permission_probe_agrees_with_a_direct_check() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let direct = tmp.path().join("direct"); + std::fs::create_dir(&direct).expect("create dir"); + std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o555)) + .expect("make read-only"); + let refused = std::fs::File::create(direct.join("file")).is_err(); + std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore"); + assert_eq!(probe_dir_permissions_bind(tmp.path()), refused); + } } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 380d8210c1..327bf31722 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -32,6 +32,7 @@ const REQUIRES_NO_GPU_TAG: &str = "requires-no-gpu"; const REQUIRES_BARE_METAL_TAG: &str = "requires-bare-metal"; const REQUIRES_WSL_TAG: &str = "requires-wsl"; const REQUIRES_CASE_SENSITIVE_FS_TAG: &str = "requires-case-sensitive-fs"; +const REQUIRES_DIR_PERMISSIONS_TAG: &str = "requires-dir-permissions"; const SERVE_TIMEOUT_PREFIX: &str = "serve-timeout:"; const NIGHTLY_TAG: &str = "nightly"; const LIFECYCLE_TAG: &str = "lifecycle"; @@ -122,6 +123,15 @@ pub struct ScenarioDecl { /// directory or a mounted Windows drive. The answer comes from the capability /// probe's `case_sensitive_fs`, measured on that same temp root. pub requires_case_sensitive_fs: bool, + /// `@requires-dir-permissions`: the scenario's premise is a directory that + /// refuses new files once its write bits are cleared, so it is skipped for a + /// user those bits do not bind — root, which is what the self-hosted WSL2 + /// guest runs the suite as. + /// + /// `@requires-os:linux` cannot express this: root on Linux is still + /// `linux`. The answer comes from the capability probe's + /// `dir_permissions_bind`, measured under the same temp root. + pub requires_dir_permissions: bool, /// Engine the scenario pins via `@requires-engine:` (if any). pub requires_engine: Option, /// OS the scenario requires via `@requires-os:` (e.g. "linux"), if any — @@ -170,6 +180,7 @@ impl ScenarioDecl { let mut requires_bare_metal = false; let mut requires_wsl = false; let mut requires_case_sensitive_fs = false; + let mut requires_dir_permissions = false; let mut requires_engine = None; let mut requires_os = None; let mut serve_timeout_secs = None; @@ -205,6 +216,8 @@ impl ScenarioDecl { requires_wsl = true; } else if tag == REQUIRES_CASE_SENSITIVE_FS_TAG { requires_case_sensitive_fs = true; + } else if tag == REQUIRES_DIR_PERMISSIONS_TAG { + requires_dir_permissions = true; } else if tag == NIGHTLY_TAG { nightly = true; } else if tag == LIFECYCLE_TAG { @@ -222,6 +235,7 @@ impl ScenarioDecl { requires_bare_metal, requires_wsl, requires_case_sensitive_fs, + requires_dir_permissions, requires_engine, requires_os, serve_timeout_secs, @@ -577,6 +591,13 @@ pub fn resolve( .to_owned(), }; } + if decl.requires_dir_permissions && !cap.dir_permissions_bind { + return Expectation::Skip { + reason: "requires directory permissions that bind the user running the suite; a \ + read-only directory still takes new files here (root?)" + .to_owned(), + }; + } let engine = decl.effective_engine(cap); // A scenario that pins or defaults to a real engine and would actually serve // needs that engine to be startable here. We treat any GPU scenario with a @@ -685,6 +706,7 @@ mod tests { effective_serve_engine: "vllm".into(), platform_slug: "mi300x".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, "strix-ubuntu" => HostCapability { os_family: "linux".into(), @@ -696,6 +718,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, "strix-windows" => HostCapability { os_family: "windows".into(), @@ -707,6 +730,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), case_sensitive_fs: false, + dir_permissions_bind: true, }, // A WSL2 dev box with the ROCm passthrough in place: `os_family` is // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes @@ -727,6 +751,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo-wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, // The same box without the passthrough: the gfx target is reported // by the Windows-side driver but ROCm cannot reach it, so the probe @@ -741,6 +766,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, // The hosted WSL runner before the ROCm passthrough is complete: // Windows still reports the gfx target, but the CLI cannot use it. @@ -754,6 +780,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo-wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, _ => HostCapability { os_family: "other".into(), @@ -765,6 +792,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "mock".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, } } @@ -1529,6 +1557,7 @@ serve_timeout_secs = 90 // so `@requires-os:linux` alone would have let the scenario run. let folded = HostCapability { case_sensitive_fs: false, + dir_permissions_bind: true, ..cap("wsl2") }; let both = decl(&[ @@ -1553,6 +1582,50 @@ serve_timeout_secs = 90 ); } + #[test] + fn dir_permissions_tag_parses_in_both_shapes() { + assert!(decl(&["id:x", "requires-dir-permissions"]).requires_dir_permissions); + assert!(decl(&["@id:x", "@requires-dir-permissions"]).requires_dir_permissions); + // Absent by default, so no existing scenario changes meaning. + assert!(!decl(&["id:x", "requires-os:linux"]).requires_dir_permissions); + } + + #[test] + fn requires_dir_permissions_skips_for_a_user_the_permission_bits_do_not_bind() { + let m = Expectations::default(); + let none = Included::default(); + let d = decl(&[ + "id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced", + "requires-os:linux", + "requires-dir-permissions", + ]); + assert_eq!( + resolve(&d, &cap("mi300x"), &m, none), + Expectation::ExpectPass, + "an ordinary user's read-only directory refuses new files, so the scenario runs" + ); + // The case the gate exists for: the WSL2 guest runs the suite as root, + // so `os_family` is `linux` and `@requires-os:linux` alone would let the + // scenario run and pass without testing anything. + let as_root = HostCapability { + dir_permissions_bind: false, + ..cap("wsl2") + }; + match resolve(&d, &as_root, &m, none) { + Expectation::Skip { reason } => assert!( + reason.contains("directory permissions"), + "the skip must name the missing premise: {reason}" + ), + other => panic!("a user the permission bits do not bind must skip, got {other:?}"), + } + // Without the tag the same host still runs it: the probe answer only + // matters to scenarios that declare the premise. + assert_eq!( + resolve(&decl(&["id:x", "requires-os:linux"]), &as_root, &m, none), + Expectation::ExpectPass + ); + } + #[test] fn os_reason_wins_over_case_folding_reason_on_a_non_matching_os() { // Native Windows fails both premises of the case-twin scenario; the grid @@ -1593,6 +1666,7 @@ reason = "unrelated open bug" ]); let folded = HostCapability { case_sensitive_fs: false, + dir_permissions_bind: true, ..cap("strix-ubuntu") }; assert!(matches!( diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 0bf3b82544..8e45c6f041 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -690,6 +690,12 @@ impl Drop for E2eWorld { if let Some(root) = &self.isolated_root { stop_managed_services(root.path()); } + // A scenario that made the audit directory read-only must give it back + // before the TempDir below tries to remove it, even if a step panicked. + #[cfg(unix)] + if self.isolated_root.is_some() { + e2e::automations_steps::restore_audit_dir_permissions(&self.app_paths().audit_dir()); + } // `isolated_root` is a `TempDir`; its own Drop removes the directory. } } diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 3a3534860d..82866ece6c 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -14,7 +14,8 @@ //! guard skips the spawn. Contracts verified against the running Linux binary //! (EAI-8072). Scoped to the enable/disable/mode slice, the proposal-history //! listing's unit formatting, the bare listing's tolerance of a torn -//! audit-log tail, and the size cap on the audit log and compaction of the +//! audit-log tail, the repair of one by the next audited command, and the +//! size cap on the audit log and compaction of the //! proposal history (ROCMAI-729); the watcher/daemon trigger behaviour itself //! is covered separately. @@ -181,6 +182,89 @@ async fn torn_audit_tail(world: &mut E2eWorld) { .expect("failed to seed a torn audit-event tail"); } +/// Make the audit log's directory refuse new files while the log inside it +/// stays writable, so the repair of a torn tail cannot write the clean copy it +/// renames into place — the failure a full disk causes, without filling one. +/// +/// The CLI's lock file for the log is created first: it is reused when it +/// exists but cannot be created in a directory that takes no new files. Fails +/// if the directory still takes new files afterwards: directory permissions do +/// not bind root, and the scenario's `@requires-dir-permissions` tag skips it +/// there, so this is the backstop that keeps it from passing without +/// exercising the fallback should the tag be dropped. +/// [`restore_audit_dir_permissions`] undoes it so the scenario's temp root can +/// be removed. +#[given("the audit log's directory takes no new files")] +async fn audit_dir_takes_no_new_files(world: &mut E2eWorld) { + let dir = world.app_paths().audit_dir(); + std::fs::File::create(dir.join("events.jsonl.lock")).expect("failed to create the lock file"); + forbid_new_files(&dir); + let probe = dir.join("probe"); + if std::fs::File::create(&probe).is_ok() { + let _ = std::fs::remove_file(&probe); + panic!( + "{} still takes new files, so this scenario cannot make the audit log's \ + replacement fail: directory permissions do not bind this user (root?)", + dir.display() + ); + } +} + +#[cfg(unix)] +fn forbid_new_files(dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o555)) + .expect("failed to make the audit directory read-only"); +} + +#[cfg(not(unix))] +fn forbid_new_files(_dir: &std::path::Path) { + panic!("this step needs Unix directory permissions; its scenario is @requires-os:linux"); +} + +/// Undo [`audit_dir_takes_no_new_files`] on `dir`, best effort, from the +/// world's teardown. +#[cfg(unix)] +pub(crate) fn restore_audit_dir_permissions(dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + if dir.is_dir() { + let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o755)); + } +} + +#[then("the CLI prints no warning about the audit log")] +async fn no_audit_warning(world: &mut E2eWorld) { + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert!( + !stderr.to_lowercase().contains("audit"), + "expected no audit-log warning, got stderr:\n{stderr}" + ); +} + +#[then( + "the audit log keeps its earlier record, drops the torn line and records the permissions change" +)] +async fn audit_log_repaired_and_appended(world: &mut E2eWorld) { + let path = world.app_paths().audit_events_path(); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + // `audit_records` parses every line, so a left-over torn fragment fails here. + let records = audit_records(&path); + let seen: Vec<(&str, &str)> = records + .iter() + .map(|record| (record.message.as_str(), record.action.as_str())) + .collect(); + assert_eq!( + seen, + [ + ("clean", "append"), + ("permissions mode set to ask", "set_mode") + ], + "unexpected audit log:\n{text}" + ); + assert!(text.ends_with('\n'), "the last record must be terminated"); +} + #[given("an enabled automation watcher")] async fn enabled_watcher(world: &mut E2eWorld) { suppress_daemon_spawn(world); From 9cbf0042896486c47980e8cd1ec59baa7b1981be Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:49:47 +0000 Subject: [PATCH 4/5] test(e2e): allow missing_const_for_fn on the non-Unix stubs of the permission gate Windows clippy flagged the non-Unix arm of the directory-permission probe as a candidate for const fn. Its Unix sibling does I/O and cannot be const, so making only one arm const would give the platforms different signatures; follow the existing platform-stub convention and allow the lint on both new stubs, with the reason. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/src/capability.rs | 3 +++ tests/e2e-cucumber/tests/e2e/automations_steps.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 244b4d04c6..abe4d87efe 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -461,6 +461,9 @@ fn probe_dir_permissions_bind(dir: &std::path::Path) -> bool { } #[cfg(not(unix))] +// Platform stub. The Unix arm does I/O and cannot be `const`, so making only +// this arm `const` would give the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] fn probe_dir_permissions_bind(_dir: &std::path::Path) -> bool { false } diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 82866ece6c..d46e7d3b67 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -218,6 +218,10 @@ fn forbid_new_files(dir: &std::path::Path) { } #[cfg(not(unix))] +// Platform stub for a `@requires-os:linux` scenario. The Unix arm does I/O and +// cannot be `const`, so making only this arm `const` would give the two +// platforms different signatures. +#[allow(clippy::missing_const_for_fn)] fn forbid_new_files(_dir: &std::path::Path) { panic!("this step needs Unix directory permissions; its scenario is @requires-os:linux"); } From 9d363722b0ae32d14771dd53d65c1b8aabe5f28c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 13:26:50 +0000 Subject: [PATCH 5/5] fix(core): address review nits on the torn-tail fallback Rewrap two comments that the earlier reflow left over width. Correct the reader-splice note: the short-read guard compares each read with the scan's current offset, so the file only has to regrow past that offset, not to its old length. Expose jsonl_lock_path so the audit-directory e2e step derives the lock file name from the same helper the writers use instead of hardcoding it. Pin that the OS reason wins over the directory-permissions reason on a non-matching OS, as it already does for the case-folding tag. Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 46 ++++++++++--------- crates/rocm-core/src/lib.rs | 6 +-- tests/e2e-cucumber/src/expectation.rs | 32 +++++++++++++ .../tests/e2e/automations_steps.rs | 8 ++-- 4 files changed, 64 insertions(+), 28 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 7b29be370c..c41af5369e 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -248,12 +248,11 @@ pub struct AuditEventRecord { /// serialize against other appends and because it can itself replace or /// rename the live file out from under a concurrent writer — a torn tail is /// repaired by renaming a new file over the store, or by truncating it when -/// that fails (see [`TailState::Torn`]), -/// and a rotating events or audit store is renamed aside to `.1` (see -/// [`rotate_jsonl_generations`]) — and an append that had already opened the -/// old file would otherwise land in the replaced or rotated-away copy and -/// vanish. -fn jsonl_lock_path(path: &Path) -> PathBuf { +/// that fails (see `TailState::Torn`), and a rotating events or audit store is +/// renamed aside to `.1` (see `rotate_jsonl_generations`) — and an +/// append that had already opened the old file would otherwise land in the +/// replaced or rotated-away copy and vanish. +pub fn jsonl_lock_path(path: &Path) -> PathBuf { let mut name = path.file_name().unwrap_or_default().to_os_string(); name.push(".lock"); path.with_file_name(name) @@ -463,10 +462,11 @@ fn append_jsonl_record( /// without closing it: a scan that comes back short is refused or retried /// (see [`ReverseLines`] and `load_recent_jsonl_records_from`), but one that /// already read part of a debris line longer than a scan chunk and then finds -/// the file regrown to at least its old length reads on undetected, and can -/// splice the two reads into a record nobody wrote. That is the cost of not -/// wedging the store, accepted for a path that needs a failing replace, a -/// debris line that long and a reader mid-scan all at once. +/// the file regrown past the offset the scan has reached (not its old length: +/// the check compares each read with that offset, which only shrinks) reads on +/// undetected, and can splice the two reads into a record nobody wrote. That +/// is the cost of not wedging the store, accepted for a path that needs a +/// failing replace, a debris line that long and a reader mid-scan all at once. /// /// If `truncate` fails too, the error names both causes and the store is left /// as it was; readers keep skipping the debris. If the append after a @@ -665,10 +665,11 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// fallback, taken only when that replacement fails and only under the store's /// lock (see [`repair_torn_tail_and_append`]); otherwise a short read here /// means the store was edited or truncated out-of-band, a case this module -/// does not try to recover from cleanly. Most callers treat what is still there the same as any other -/// unexpectedly-ending tail; [`ReverseLines`] is the one exception, and -/// reports an error instead once it would have to splice such a read onto -/// bytes it already read, rather than risk inventing a line. +/// does not try to recover from cleanly. Most callers treat what is still +/// there the same as any other unexpectedly-ending tail; [`ReverseLines`] is +/// the one exception, and reports an error instead once it would have to +/// splice such a read onto bytes it already read, rather than risk inventing +/// a line. fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result> { let mut reader = file; reader @@ -2328,14 +2329,15 @@ mod tests { } // Nothing in this module's own writers shrinks a store (bar the torn-tail - // repair's rare fallback), but an out-of-band edit might, and load_recent_jsonl_records is supposed to - // retry from the new length rather than fail. Driving that needs the - // shrink to land strictly between two of ReverseLines's own chunked - // reads (see the `test_hooks` module doc) — which in turn - // needs a scan that takes more than one physical read in the first - // place, hence the pathologically long first record here (same - // technique as append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly - // below, just complete JSON rather than torn). + // repair's rare fallback), but an out-of-band edit might, and + // load_recent_jsonl_records is supposed to retry from the new length + // rather than fail. Driving that needs the shrink to land strictly + // between two of ReverseLines's own chunked reads (see the `test_hooks` + // module doc) — which in turn needs a scan that takes more than one + // physical read in the first place, hence the pathologically long first + // record here (same technique as + // append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly below, + // just complete JSON rather than torn). #[test] fn load_recent_retries_past_a_shrink_mid_scan_and_succeeds() -> Result<()> { let (root, paths) = temp_app_paths("scan-retry-after-shrink"); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f3b7cc7f76..1685a9fc1e 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -60,9 +60,9 @@ pub use automation::{ AutomationTriggerEvent, BuiltinWatcherSpec, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, WatcherRuntimeSnapshot, append_audit_event, append_automation_event, append_automation_proposal, builtin_watcher, - builtin_watchers, find_automation_proposal, generate_proposal_id, load_automation_proposals, - load_recent_audit_events, load_recent_automation_events, load_recent_automation_proposals, - update_automation_proposal_status, + builtin_watchers, find_automation_proposal, generate_proposal_id, jsonl_lock_path, + load_automation_proposals, load_recent_audit_events, load_recent_automation_events, + load_recent_automation_proposals, update_automation_proposal_status, }; pub use binary_paths::{daemon_binary_path, engine_binary_path, sibling_binary_path}; pub use codex_bridge::{CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot}; diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 327bf31722..073ac85baa 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -1647,6 +1647,38 @@ serve_timeout_secs = 90 } } + #[test] + fn os_reason_wins_over_dir_permissions_reason_on_a_non_matching_os() { + // The same ordering as the case-folding twin above, for the tag the + // torn-audit-tail scenario carries next to `@requires-os:linux`. + let m = Expectations::default(); + let d = decl(&[ + "id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced", + "requires-os:linux", + "requires-dir-permissions", + ]); + assert!( + d.requires_dir_permissions && d.requires_os.is_some(), + "both premise tags must parse, or the ordering below proves nothing" + ); + let host = HostCapability { + dir_permissions_bind: false, + ..cap("strix-windows") + }; + assert_ne!(host.os_family, "linux", "fixture must fail the OS premise"); + assert!( + !host.dir_permissions_bind, + "fixture must fail both premises" + ); + match resolve(&d, &host, &m, Included::default()) { + Expectation::Skip { reason } => assert!( + reason.contains("requires os 'linux'"), + "the OS reason must win: {reason}" + ), + other => panic!("native Windows must skip, got {other:?}"), + } + } + #[test] fn case_folding_skip_is_not_recorded_as_a_known_bug() { // A host that cannot hold the premise is not-applicable, not a defect, so diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index d46e7d3b67..0204fe8bc4 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -23,7 +23,7 @@ use cucumber::{given, then, when}; use rocm_core::{ AuditEventRecord, AutomationProposalRecord, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, append_automation_proposal, - load_automation_proposals, + jsonl_lock_path, load_automation_proposals, }; use crate::E2eWorld; @@ -196,8 +196,10 @@ async fn torn_audit_tail(world: &mut E2eWorld) { /// be removed. #[given("the audit log's directory takes no new files")] async fn audit_dir_takes_no_new_files(world: &mut E2eWorld) { - let dir = world.app_paths().audit_dir(); - std::fs::File::create(dir.join("events.jsonl.lock")).expect("failed to create the lock file"); + let paths = world.app_paths(); + let dir = paths.audit_dir(); + std::fs::File::create(jsonl_lock_path(&paths.audit_events_path())) + .expect("failed to create the lock file"); forbid_new_files(&dir); let probe = dir.join("probe"); if std::fs::File::create(&probe).is_ok() {