diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index f1a286ebd6..c41af5369e 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -19,7 +19,10 @@ //! its own new record as a single rewrite via `write_atomic`, so a lock-free //! reader never observes a file shrunk mid-repair. That rewrite costs O(store) //! rather than O(record), the same as a proposal status update, which rewrites -//! the whole store via `write_atomic` too. Nothing in this module ever +//! the whole store via `write_atomic` too. Only when that rewrite itself fails +//! (a full disk, or on Windows another program holding the store open without +//! delete sharing) does the append cut the debris in place instead, so a store +//! cannot be wedged by a failed repair; otherwise nothing in this module ever //! shrinks a store in place. Readers take no lock, and an unterminated last //! line that is not valid JSON — an append caught mid-write — is skipped //! rather than reported as corruption. @@ -244,12 +247,12 @@ pub struct AuditEventRecord { /// rewrite needs it for its read-modify-write; an append needs it both to /// serialize against other appends and because it can itself replace or /// rename the live file out from under a concurrent writer — a torn tail is -/// repaired by renaming a new file over the store (see [`TailState::Torn`]), -/// and a rotating events or audit store is renamed aside to `.1` (see -/// [`rotate_jsonl_generations`]) — and an append that had already opened the -/// old file would otherwise land in the replaced or rotated-away copy and -/// vanish. -fn jsonl_lock_path(path: &Path) -> PathBuf { +/// repaired by renaming a new file over the store, or by truncating it when +/// that fails (see `TailState::Torn`), and a rotating events or audit store is +/// renamed aside to `.1` (see `rotate_jsonl_generations`) — and an +/// append that had already opened the old file would otherwise land in the +/// replaced or rotated-away copy and vanish. +pub fn jsonl_lock_path(path: &Path) -> PathBuf { let mut name = path.file_name().unwrap_or_default().to_os_string(); name.push(".lock"); path.with_file_name(name) @@ -367,8 +370,9 @@ fn open_jsonl_for_append(path: &Path) -> Result { /// back, and the debris itself is discarded by the *next* append rather than /// being cut first as a separate step — see [`inspect_tail`] and /// [`TailState::Torn`]. A reader already skips the debris in the meantime. -/// Nothing in this function ever truncates the live file, so a lock-free -/// reader never observes it shrink. +/// This function truncates the live file only as the last resort when that +/// replacement itself fails (see [`repair_torn_tail_and_append`]), so a +/// lock-free reader can observe it shrink only then, never on the normal path. /// /// [`append_or_tolerate_a_missing_newline`] checks whether a failed write /// landed anyway before reporting it, so a write that fails but still lands @@ -419,31 +423,14 @@ fn append_jsonl_record( } } match inspect_tail(path, &file)? { - TailState::Torn(tail_start) => { - // A torn tail is debris — crash or a failed write — that must be - // cut before this record lands. Cutting it in place and then - // appending as a second step would briefly shrink the file for - // any lock-free reader mid-scan, and regrowing it afterward can - // make that scan splice bytes read at two different moments into - // something that parses as a record nobody wrote (the hazard - // `ReverseLines::next_line`'s short-read guard exists for). - // Replacing the whole file atomically instead means a reader - // only ever sees the pre-repair length or this append's, never a - // length in between. - let mut content = read_range(path, &file, 0, tail_start)?; - content.extend_from_slice(&line); - // Done with the pre-repair file's handle before replacing it, - // rather than leaving that to whatever happens to still have - // `file` open at function return. The rename already succeeds - // with this handle open regardless — std opens files with - // `FILE_SHARE_DELETE` by default on Windows, so a rename isn't - // blocked by it (see `write_atomic`'s doc for the case that - // *isn't* covered: another process holding the destination open - // without delete-sharing) — but this states the intent rather - // than leaning on that default. - drop(file); - crate::write_atomic(path, &content, "jsonl")?; - } + TailState::Torn(tail_start) => repair_torn_tail_and_append( + path, + file, + tail_start, + &line, + |path, content| crate::write_atomic(path, content, "jsonl"), + truncate_jsonl_in_place, + )?, TailState::ValidUnterminated(start) => { line.insert(0, b'\n'); append_or_tolerate_a_missing_newline(path, &file, &line, start)?; @@ -455,6 +442,94 @@ fn append_jsonl_record( Ok(()) } +/// Discard the torn tail of the store at `path`, which starts at `tail_start`, +/// and land `line` in its place. The caller must hold the store's +/// [`jsonl_lock_path`] lock; `file` is its handle on the pre-repair store. +/// +/// The repair publishes the clean prefix plus `line` as one `replace`, so a +/// lock-free reader only ever sees the pre-repair length or the post-append +/// length (see [`append_jsonl_record`]). That rewrite can itself fail — no +/// room for the copy on a full disk, or on Windows another program holding +/// the store open without delete sharing — and leaving it there would wedge +/// the store: every later append would hit the same debris and the same +/// failure. Only then does this fall back to `truncate`, cutting the debris +/// in place and appending `line` as a plain append. A truncate frees space +/// rather than needing it and renames nothing, so it can succeed where +/// `replace` cannot. Like a repair by `replace`, the fallback prints nothing. +/// +/// The fallback reopens, on this already-failing path only, the shrink a +/// lock-free reader can observe, and the reader's guards narrow that window +/// without closing it: a scan that comes back short is refused or retried +/// (see [`ReverseLines`] and `load_recent_jsonl_records_from`), but one that +/// already read part of a debris line longer than a scan chunk and then finds +/// the file regrown past the offset the scan has reached (not its old length: +/// the check compares each read with that offset, which only shrinks) reads on +/// undetected, and can splice the two reads into a record nobody wrote. That +/// is the cost of not wedging the store, accepted for a path that needs a +/// failing replace, a debris line that long and a reader mid-scan all at once. +/// +/// If `truncate` fails too, the error names both causes and the store is left +/// as it was; readers keep skipping the debris. If the append after a +/// successful truncate fails, the error carries the replace failure too: it is +/// the root cause. +/// +/// `replace` and `truncate` are parameters so a test can make either fail on +/// demand. +fn repair_torn_tail_and_append( + path: &Path, + file: fs::File, + tail_start: u64, + line: &[u8], + replace: impl FnOnce(&Path, &[u8]) -> Result<()>, + truncate: impl FnOnce(&Path, u64) -> Result<()>, +) -> Result<()> { + let mut content = read_range(path, &file, 0, tail_start)?; + content.extend_from_slice(line); + // Done with the pre-repair file's handle before replacing it, rather than + // leaving that to whatever happens to still have `file` open at function + // return. The rename already succeeds with this handle open regardless — + // std opens files with `FILE_SHARE_DELETE` by default on Windows, so a + // rename isn't blocked by it (see `write_atomic`'s doc for the case that + // *isn't* covered: another process holding the destination open without + // delete-sharing) — but this states the intent rather than leaning on that + // default. + drop(file); + let replace_error = match replace(path, &content) { + Ok(()) => return Ok(()), + Err(error) => error, + }; + drop(content); + if let Err(truncate_error) = truncate(path, tail_start) { + bail!( + "failed to repair the torn tail of {}: replacing it failed ({replace_error:#}); \ + truncating it failed ({truncate_error:#})", + path.display() + ); + } + open_jsonl_for_append(path) + .and_then(|file| append_or_tolerate_a_missing_newline(path, &file, line, tail_start)) + .with_context(|| { + format!( + "failed to append to {} after cutting its torn tail \ + (replacing the store had failed: {replace_error:#})", + path.display() + ) + }) +} + +/// Cut the store at `path` down to `len` bytes through a fresh write handle. +/// +/// Not the append handle: on Windows an append-only handle lacks +/// `FILE_WRITE_DATA`, which `set_len` needs. +fn truncate_jsonl_in_place(path: &Path, len: u64) -> Result<()> { + fs::OpenOptions::new() + .write(true) + .open(path) + .with_context(|| format!("failed to open {} for writing", path.display()))? + .set_len(len) + .with_context(|| format!("failed to truncate {}", path.display())) +} + /// The warning to print that rotating `path` failed, or `None` after the first /// one for that store in this process, so a rename that keeps failing (e.g. /// another program holds the live file open without delete sharing on @@ -545,8 +620,9 @@ enum TailState { ValidUnterminated(u64), /// Debris — crash, or a write that failed partway through a genuinely /// incomplete record — starting at this offset. Must be cut before the - /// next record lands, but never by truncating in place — see - /// [`append_jsonl_record`]. + /// next record lands, by replacing the store rather than truncating it in + /// place — except as the fallback when that replacement fails; see + /// [`append_jsonl_record`] and [`repair_torn_tail_and_append`]. Torn(u64), } @@ -583,15 +659,17 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// Read bytes `[start, end)` of `file`. /// /// Returns fewer bytes if the file now ends before `end`. Nothing this -/// module's own writers do can shrink a store in place — an append only ever +/// module's own writers do shrinks a store in place — an append only ever /// grows it, and a torn-tail repair replaces it wholesale via `write_atomic` -/// rather than truncating the live file (see [`inspect_tail`] and -/// [`TailState::Torn`]) — so a short read here means the store was edited or -/// truncated out-of-band, a case this module does not try to recover from -/// cleanly. Most callers treat what is still there the same as any other -/// unexpectedly-ending tail; [`ReverseLines`] is the one exception, and -/// reports an error instead once it would have to splice such a read onto -/// bytes it already read, rather than risk inventing a line. +/// (see [`inspect_tail`] and [`TailState::Torn`]) — except the repair's +/// fallback, taken only when that replacement fails and only under the store's +/// lock (see [`repair_torn_tail_and_append`]); otherwise a short read here +/// means the store was edited or truncated out-of-band, a case this module +/// does not try to recover from cleanly. Most callers treat what is still +/// there the same as any other unexpectedly-ending tail; [`ReverseLines`] is +/// the one exception, and reports an error instead once it would have to +/// splice such a read onto bytes it already read, rather than risk inventing +/// a line. fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result> { let mut reader = file; reader @@ -693,8 +771,8 @@ fn load_recent_jsonl_records( /// A rotation can rename files between two of the opens: opening the live /// file and then `.1` across one would open the same file twice (and read its /// records twice), and opening `.1` and then the live file across one would -/// skip a generation. A torn-tail repair (see [`TailState::Torn`]) also -/// replaces the live file, by renaming a new one over it — not a rotation, +/// skip a generation. A torn-tail repair (see [`TailState::Torn`]) normally +/// also replaces the live file, by renaming a new one over it — not a rotation, /// but the same hazard for this snapshot: a path opened before the repair /// would still read the pre-repair file by handle, yet a path opened after /// would see different content at the same name. So after opening them all, @@ -775,10 +853,13 @@ fn open_jsonl_generations(path: &Path, generations: usize) -> Result( path: &Path, file: &fs::File, @@ -923,10 +1004,10 @@ impl<'a> ReverseLines<'a> { // A short read while `pending` is empty — nothing accumulated // yet, whether from an earlier call or from this one — just // means the store was edited or truncated out-of-band (see - // `read_range`); nothing a concurrent writer does can shrink it - // in place. Once `pending` holds any bytes, a short read means - // they no longer continue from it, so splicing them would - // invent a line. + // `read_range`); no concurrent writer shrinks it in place short of + // the torn-tail repair's rare fallback. Once `pending` holds any + // bytes, a short read means they no longer continue from it, so + // splicing them would invent a line. if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { bail!("{} shrank while it was being read", self.path.display()); } @@ -2247,15 +2328,16 @@ mod tests { Ok(()) } - // Nothing in this module's own writers shrinks a store, but an - // out-of-band edit might, and load_recent_jsonl_records is supposed to - // retry from the new length rather than fail. Driving that needs the - // shrink to land strictly between two of ReverseLines's own chunked - // reads (see the `test_hooks` module doc) — which in turn - // needs a scan that takes more than one physical read in the first - // place, hence the pathologically long first record here (same - // technique as append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly - // below, just complete JSON rather than torn). + // Nothing in this module's own writers shrinks a store (bar the torn-tail + // repair's rare fallback), but an out-of-band edit might, and + // load_recent_jsonl_records is supposed to retry from the new length + // rather than fail. Driving that needs the shrink to land strictly + // between two of ReverseLines's own chunked reads (see the `test_hooks` + // module doc) — which in turn needs a scan that takes more than one + // physical read in the first place, hence the pathologically long first + // record here (same technique as + // append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly below, + // just complete JSON rather than torn). #[test] fn load_recent_retries_past_a_shrink_mid_scan_and_succeeds() -> Result<()> { let (root, paths) = temp_app_paths("scan-retry-after-shrink"); @@ -2476,6 +2558,286 @@ mod tests { Ok(()) } + /// The store's file identity, as a string so one assertion covers the + /// inode on Unix and the volume serial plus file index on Windows. + #[cfg(unix)] + fn file_identity(path: &Path) -> Result { + use std::os::unix::fs::MetadataExt; + Ok(fs::metadata(path)?.ino().to_string()) + } + + #[cfg(windows)] + fn file_identity(path: &Path) -> Result { + Ok(format!( + "{:?}", + crate::test_support::windows_file_identity(path)? + )) + } + + /// Run the torn-tail repair on the seeded audit store with the replace and + /// truncate steps supplied by the test, appending the record `b`. + fn repair_audit_store_with( + paths: &AppPaths, + replace: impl FnOnce(&Path, &[u8]) -> Result<()>, + truncate: impl FnOnce(&Path, u64) -> Result<()>, + ) -> Result<()> { + let path = paths.audit_events_path(); + let file = open_jsonl_for_append(&path)?; + let TailState::Torn(tail_start) = inspect_tail(&path, &file)? else { + bail!("the seeded store has no torn tail"); + }; + let mut line = audit_line("b").into_bytes(); + line.push(b'\n'); + repair_torn_tail_and_append(&path, file, tail_start, &line, replace, truncate) + } + + // The fallback the whole-store rewrite lacked: when it cannot complete + // (a full disk, a sharing violation), the debris is cut in place instead + // of wedging every later append on it. The same file identity afterwards + // is what tells that apart from the replace the test above pins. + #[test] + fn a_torn_tail_is_cut_in_place_when_replacing_the_store_fails() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-truncates"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + let mut attempted = Vec::new(); + + repair_audit_store_with( + &paths, + |_, content| { + attempted = content.to_vec(); + bail!("no space left on device") + }, + truncate_jsonl_in_place, + )?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + let expected = format!("{}\n{}\n", audit_line("a"), audit_line("b")); + assert_eq!( + String::from_utf8(attempted)?, + expected, + "the whole-store replace must be tried first, with the full repaired store" + ); + assert_eq!(text?, expected); + assert_eq!(before, after, "the fallback must cut the debris in place"); + assert_eq!(audit_messages(&recent?), ["a", "b"]); + Ok(()) + } + + // The fallback is a last resort: a replace that works must not be + // followed by a truncate, or the shrink window it closes would be back on + // every repair. + #[test] + fn a_torn_tail_is_not_truncated_when_replacing_the_store_works() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-unused"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + + repair_audit_store_with( + &paths, + |path, content| crate::write_atomic(path, content, "jsonl"), + |_, _| bail!("truncate must not run when the replace succeeded"), + )?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_ne!(before, after, "a working replace publishes a new file"); + Ok(()) + } + + #[test] + fn a_torn_tail_that_can_be_neither_replaced_nor_cut_reports_both_causes() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-both-fail"); + let torn = &audit_line("torn")[..20]; + let seeded = format!("{}\n{torn}", audit_line("a")); + seed_audit_store(&paths, &seeded)?; + let path = paths.audit_events_path(); + + // Layered like the real steps (`write_atomic` and + // `truncate_jsonl_in_place` both wrap the OS error in a context), so + // the assertion fails if either cause is formatted without its chain. + let result = repair_audit_store_with( + &paths, + |_, _| { + Err(anyhow::anyhow!("no space left on device")) + .context("failed to replace the store") + }, + |_, _| { + Err(anyhow::anyhow!("permission denied")).context("failed to truncate the store") + }, + ); + + let text = fs::read_to_string(&path); + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert_eq!( + format!("{:#}", result.expect_err("both steps failed")), + format!( + "failed to repair the torn tail of {}: replacing it failed (failed to replace \ + the store: no space left on device); truncating it failed (failed to truncate \ + the store: permission denied)", + path.display() + ) + ); + assert_eq!( + text?, seeded, + "a failed repair must leave the store as it was" + ); + assert_eq!(audit_messages(&recent?), ["a"]); + Ok(()) + } + + // The replace failure is the root cause, so it must survive into the error + // when the cut itself works but the append after it does not. The truncate + // stub swaps the store for a directory, which makes reopening it fail. + #[test] + fn a_failed_append_after_the_cut_still_reports_why_the_replace_failed() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-append-fails"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + + let result = repair_audit_store_with( + &paths, + |_, _| { + Err(anyhow::anyhow!("no space left on device")) + .context("failed to replace the store") + }, + |path, _| { + fs::remove_file(path)?; + fs::create_dir(path)?; + Ok(()) + }, + ); + + fs::remove_dir_all(root).ok(); + let message = format!("{:#}", result.expect_err("the append after the cut failed")); + assert!( + message.starts_with(&format!( + "failed to append to {} after cutting its torn tail (replacing the store had \ + failed: failed to replace the store: no space left on device): ", + path.display() + )), + "unexpected error: {message}" + ); + Ok(()) + } + + // After the fallback the store is clean, so the next append is the plain + // one-line append: no second repair, no new file. + #[test] + fn appends_after_a_fallback_repair_are_plain_appends() -> Result<()> { + let (root, paths) = temp_app_paths("torn-tail-fallback-then-append"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + repair_audit_store_with( + &paths, + |_, _| bail!("no space left on device"), + truncate_jsonl_in_place, + )?; + let repaired = file_identity(&path)?; + + append_audit_event(&paths, &audit_record("c"))?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!( + text?, + format!( + "{}\n{}\n{}\n", + audit_line("a"), + audit_line("b"), + audit_line("c") + ) + ); + assert_eq!(repaired, after); + Ok(()) + } + + // The real trigger rather than an injected one: a store directory that + // takes no new files, so the temp copy of the whole-store replace cannot + // be created, while the store file itself can still be written. + // + // This is the only test that drives the production `replace`/`truncate` + // wiring in `append_jsonl_record` rather than injected steps, and it passes + // without testing anything for a user the permission bits do not bind + // (root). CI runs it as a regular user. + #[cfg(unix)] + #[test] + fn append_repairs_a_torn_tail_in_a_directory_that_takes_no_new_files() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + let (root, paths) = temp_app_paths("torn-tail-read-only-dir"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let dir = path.parent().context("the audit store has a directory")?; + // Create the lock file now: the append reuses an existing one, but + // could not create it inside the read-only directory. + drop(FileLock::acquire(jsonl_lock_path(&path))?); + let before = file_identity(&path)?; + + fs::set_permissions(dir, fs::Permissions::from_mode(0o555))?; + let read_only_holds = fs::File::create(dir.join("probe")).is_err(); + let result = read_only_holds.then(|| append_audit_event(&paths, &audit_record("b"))); + fs::set_permissions(dir, fs::Permissions::from_mode(0o755))?; + + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + let Some(result) = result else { + // Running as a user the permission bits do not bind (root). + eprintln!("skipped: a read-only directory still takes new files for this user"); + return Ok(()); + }; + result?; + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(before, after); + Ok(()) + } + + // Windows' real trigger: another program holds the store open without + // delete sharing (a virus scanner or indexer, say), so the rename over it + // is refused while the store itself can still be written. + #[cfg(windows)] + #[test] + fn append_repairs_a_torn_tail_while_another_handle_blocks_replacing_the_store() -> Result<()> { + use std::os::windows::fs::OpenOptionsExt; + + const FILE_SHARE_READ_WRITE: u32 = 0x1 | 0x2; + let (root, paths) = temp_app_paths("torn-tail-share-violation"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = file_identity(&path)?; + let blocker = fs::OpenOptions::new() + .read(true) + .share_mode(FILE_SHARE_READ_WRITE) + .open(&path)?; + + let result = append_audit_event(&paths, &audit_record("b")); + + drop(blocker); + let after = file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + result?; + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(before, after); + Ok(()) + } + #[cfg(unix)] #[test] fn append_to_a_clean_or_unterminated_store_keeps_the_same_file() -> Result<()> { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f3b7cc7f76..1685a9fc1e 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -60,9 +60,9 @@ pub use automation::{ AutomationTriggerEvent, BuiltinWatcherSpec, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, WatcherRuntimeSnapshot, append_audit_event, append_automation_event, append_automation_proposal, builtin_watcher, - builtin_watchers, find_automation_proposal, generate_proposal_id, load_automation_proposals, - load_recent_audit_events, load_recent_automation_events, load_recent_automation_proposals, - update_automation_proposal_status, + builtin_watchers, find_automation_proposal, generate_proposal_id, jsonl_lock_path, + load_automation_proposals, load_recent_audit_events, load_recent_automation_events, + load_recent_automation_proposals, update_automation_proposal_status, }; pub use binary_paths::{daemon_binary_path, engine_binary_path, sibling_binary_path}; pub use codex_bridge::{CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot}; diff --git a/docs/testing.md b/docs/testing.md index 782e892e75..d791a2cd4d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1500,9 +1500,18 @@ rotation policy; two further tests go through the public `append_automation_event`/`append_audit_event` and `load_recent_automation_events`/`load_recent_audit_events` wrappers instead, at the real size cap, so the wiring between them and the rotation policy -itself stays covered. The user-visible side is `automations-07` through -`automations-09` in +itself stays covered. The repair of a torn tail, which normally replaces the +store and falls back to cutting the debris in place when that replacement +fails, has its own tests there: each step injected to fail, the error naming +both causes when both fail, and a real fault on each platform (a directory +that takes no new files on Unix, a handle without delete sharing on Windows). +The user-visible side is `automations-07` through `automations-09`, plus +`automations-10` for the fallback, in [automations.feature](../tests/e2e-cucumber/features/automations.feature). +`automations-10` makes the audit directory read-only, which does not bind root, +so it carries `@requires-dir-permissions` and resolves to a skip for a user the +permission bits do not bind (the self-hosted WSL2 guest runs as root) rather +than pass without testing the fallback. Natural-language plan handoff tests: diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index a9852966e3..0834c45eb3 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -141,6 +141,7 @@ Scenarios carry stable-id and capability tags: | `@requires-multi-gpu` | Premise is a host with **more than one** AMD GPU present. Resolves to **skip** wherever the probed device count is not known to exceed one: a single-GPU host such as Strix Halo, and any host whose count could not be read (non-Linux, or WSL, which exposes neither a KFD topology nor an amdgpu DRM card). `@requires-gpu` cannot express this — it asks only whether *a* device is usable — so scenarios carry both. The count comes from the capability probe's `amd_gpu_count`, which mirrors the product's KFD+DRM rule rather than counting `amd-smi list` — the gate must ask the same "how many devices are present" question `--gpu` validation is built on, and `amd-smi` is only a best-effort fallback that may be absent or disagree. | | `@requires-bare-metal` | Premise is a host running the in-tree amdgpu driver, so it does not hold under WSL2. Resolves to **skip** there. `@requires-os:linux` cannot express this: WSL2 reports an `os_family` of `linux`. | | `@requires-wsl` | The inverse: premise **is** a WSL2 host. Resolves to **skip** on native Linux, native Windows, and everything else. | +| `@requires-dir-permissions` | Premise is a directory that refuses new files once its write bits are cleared. Resolves to **skip** for a user those bits do not bind: root, which is what the self-hosted WSL2 guest runs the suite as. The capability probe's `dir_permissions_bind` clears the write bits on a directory under the temp root and tries to create a file in it. `@requires-os:linux` cannot express this: root on Linux is still `linux`, and the scenario would pass without testing anything. Always **skip** on Windows, which has no such bits. | | `@requires-case-sensitive-fs` | Premise is two files whose names differ only in letter case. Resolves to **skip** where the scenarios' temp root folds case. The capability probe's `case_sensitive_fs` writes two such names in its own temp dir, which follows `$TMPDIR` like every scenario's isolated root, and checks they stayed two files. `@requires-os:linux` cannot express this: `$TMPDIR` on Linux, WSL2 included, can sit on a casefolded directory or a mounted Windows drive. | | `@requires-engine:` | Pins the serve engine. Resolves to skip where that engine can't start (e.g. vLLM on a lemonade-only Strix host). | | `@requires-os:` | Premise is OS-specific; skip on other OSes. | diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index 372846979c..fc08c39d1c 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -6,7 +6,8 @@ Feature: Automation watchers # per-entry approve/reject controls are gated on "pending"). This feature # pins the enable/disable/mode-confirmation slice, the proposal-history # listing's unit formatting, the bare listing's tolerance of a torn - # audit-log tail, and how the audit log and proposal history are kept + # audit-log tail, how the next write repairs one, and how the audit log + # and proposal history are kept # bounded; the watcher/daemon trigger behaviour itself is covered # elsewhere. Config-only (no GPU, no network), so it runs on the mock lane # every PR. @@ -127,3 +128,26 @@ Feature: Automation watchers When the user selects a permissions mode Then each rotated generation holds what the previous one held And the oldest previous generation is gone + + # A write that was cut off leaves a torn last line, and the next append + # repairs it by writing a clean copy of the audit log beside it and renaming + # that over the original. When the copy cannot be made — a full disk, or on + # Windows another program holding the log open without delete sharing — the + # repair used to give up: the debris stayed, the CLI warned that it failed to + # write the audit event on every command that records one, and the event was + # lost. The repair now falls back to cutting the debris in place. A directory + # that takes no new files is the same failure without filling a disk, so this + # drives it through `rocm config set-permissions`, a command the CLI audits. + # The fault is a directory permission, which does not bind root (the + # self-hosted WSL2 guest runs the suite as root), so `@requires-dir-permissions` + # skips it there rather than let it pass without testing anything. The unit + # tests in `automation.rs` cover the other triggers (an injected failure of + # each step, and on Windows a handle without delete sharing). + @id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced @requires-os:linux @requires-dir-permissions + Scenario: automations-10 - A command still records its audit event when the torn audit log cannot be replaced + Given the audit event log ends in a torn, unterminated line + And the audit log's directory takes no new files + When the user selects a permissions mode + Then the CLI confirms the permissions mode + And the CLI prints no warning about the audit log + And the audit log keeps its earlier record, drops the torn line and records the permissions change diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 3f4a037276..abe4d87efe 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -151,6 +151,10 @@ pub struct HostCapability { /// the temp root the scenarios' isolated roots are created in. Gates /// `@requires-case-sensitive-fs`; see [`probe_case_sensitive_fs`]. pub case_sensitive_fs: bool, + /// Whether a directory made read-only refuses new files to the user + /// running the suite. Gates `@requires-dir-permissions`; see + /// [`probe_dir_permissions_bind`]. + pub dir_permissions_bind: bool, } impl HostCapability { @@ -411,6 +415,7 @@ fn probe_host_capability() -> HostCapability { let platform_slug = derive_platform_slug(has_amd_gpu, gfx_target.as_deref(), &os_family, is_wsl); let case_sensitive_fs = probe_case_sensitive_fs(root); + let dir_permissions_bind = probe_dir_permissions_bind(root); HostCapability { os_family, @@ -422,9 +427,47 @@ fn probe_host_capability() -> HostCapability { effective_serve_engine, platform_slug, case_sensitive_fs, + dir_permissions_bind, } } +/// Whether a directory with its write bits cleared refuses to create a file for +/// the user running the suite. +/// +/// Probed rather than inferred from the OS: root is not bound by directory +/// permissions, and the self-hosted WSL2 guest runs the suite as root even +/// though it is `os_family` `linux`. A scenario whose fault is "this directory +/// takes no new files" would then pass without exercising anything, so it is +/// skipped where this answers `false`. +/// +/// The probe's directory is created under `dir`, the temp root the scenarios' +/// isolated roots follow, so it answers for the filesystem they write to, and +/// its permissions are restored before it returns so the temp root can be +/// removed. Any I/O error answers `false`: the premise has not been shown to +/// hold. Always `false` where there are no Unix permission bits to clear. +#[cfg(unix)] +fn probe_dir_permissions_bind(dir: &std::path::Path) -> bool { + use std::os::unix::fs::PermissionsExt; + + let probe = || -> std::io::Result { + let probe_dir = dir.join("permission-probe"); + std::fs::create_dir_all(&probe_dir)?; + std::fs::set_permissions(&probe_dir, std::fs::Permissions::from_mode(0o555))?; + let created = std::fs::File::create(probe_dir.join("probe-file")); + std::fs::set_permissions(&probe_dir, std::fs::Permissions::from_mode(0o755))?; + Ok(created.is_err()) + }; + probe().unwrap_or(false) +} + +#[cfg(not(unix))] +// Platform stub. The Unix arm does I/O and cannot be `const`, so making only +// this arm `const` would give the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn probe_dir_permissions_bind(_dir: &std::path::Path) -> bool { + false +} + /// Whether `dir`'s filesystem keeps two names that differ only in letter case /// apart. /// @@ -836,6 +879,7 @@ mod tests { effective_serve_engine: "lemonade".to_owned(), platform_slug: "strix-halo".to_owned(), case_sensitive_fs: false, + dir_permissions_bind: true, }; assert!(strix.engine_available("lemonade")); // vLLM adapter is "built-in" but cannot start on Windows / non-dcgpu. @@ -851,6 +895,7 @@ mod tests { effective_serve_engine: "vllm".to_owned(), platform_slug: "mi300x".to_owned(), case_sensitive_fs: true, + dir_permissions_bind: true, }; assert!(mi300x.engine_available("vllm")); assert!(mi300x.engine_available("lemonade")); @@ -1168,4 +1213,48 @@ Local model engines std::fs::write(¬_a_dir, "").expect("write file"); assert!(!probe_case_sensitive_fs(¬_a_dir)); } + + /// A probe that cannot even make its directory has not shown the premise + /// holds, so the gate must skip rather than let the scenario fail. + #[test] + fn permission_probe_answers_false_when_it_cannot_write() { + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let not_a_dir = tmp.path().join("plain-file"); + std::fs::write(¬_a_dir, "").expect("write file"); + assert!(!probe_dir_permissions_bind(¬_a_dir)); + } + + /// The probe's directory is read-only while it probes; it must be given + /// back, or the temp root it lives in could not be removed. + #[cfg(unix)] + #[test] + fn permission_probe_leaves_its_directory_removable() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + probe_dir_permissions_bind(tmp.path()); + let mode = std::fs::metadata(tmp.path().join("permission-probe")) + .expect("probe dir") + .permissions() + .mode(); + assert_eq!(mode & 0o200, 0o200, "owner write bit was not restored"); + } + + /// The answer the gated scenario relies on: where the suite runs as an + /// ordinary user, a read-only directory refuses new files. Skipped as root, + /// for whom the probe is rightly `false`. + #[cfg(unix)] + #[test] + fn permission_probe_agrees_with_a_direct_check() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::TempDir::with_prefix("capability-").expect("temp dir"); + let direct = tmp.path().join("direct"); + std::fs::create_dir(&direct).expect("create dir"); + std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o555)) + .expect("make read-only"); + let refused = std::fs::File::create(direct.join("file")).is_err(); + std::fs::set_permissions(&direct, std::fs::Permissions::from_mode(0o755)).expect("restore"); + assert_eq!(probe_dir_permissions_bind(tmp.path()), refused); + } } diff --git a/tests/e2e-cucumber/src/expectation.rs b/tests/e2e-cucumber/src/expectation.rs index 380d8210c1..073ac85baa 100644 --- a/tests/e2e-cucumber/src/expectation.rs +++ b/tests/e2e-cucumber/src/expectation.rs @@ -32,6 +32,7 @@ const REQUIRES_NO_GPU_TAG: &str = "requires-no-gpu"; const REQUIRES_BARE_METAL_TAG: &str = "requires-bare-metal"; const REQUIRES_WSL_TAG: &str = "requires-wsl"; const REQUIRES_CASE_SENSITIVE_FS_TAG: &str = "requires-case-sensitive-fs"; +const REQUIRES_DIR_PERMISSIONS_TAG: &str = "requires-dir-permissions"; const SERVE_TIMEOUT_PREFIX: &str = "serve-timeout:"; const NIGHTLY_TAG: &str = "nightly"; const LIFECYCLE_TAG: &str = "lifecycle"; @@ -122,6 +123,15 @@ pub struct ScenarioDecl { /// directory or a mounted Windows drive. The answer comes from the capability /// probe's `case_sensitive_fs`, measured on that same temp root. pub requires_case_sensitive_fs: bool, + /// `@requires-dir-permissions`: the scenario's premise is a directory that + /// refuses new files once its write bits are cleared, so it is skipped for a + /// user those bits do not bind — root, which is what the self-hosted WSL2 + /// guest runs the suite as. + /// + /// `@requires-os:linux` cannot express this: root on Linux is still + /// `linux`. The answer comes from the capability probe's + /// `dir_permissions_bind`, measured under the same temp root. + pub requires_dir_permissions: bool, /// Engine the scenario pins via `@requires-engine:` (if any). pub requires_engine: Option, /// OS the scenario requires via `@requires-os:` (e.g. "linux"), if any — @@ -170,6 +180,7 @@ impl ScenarioDecl { let mut requires_bare_metal = false; let mut requires_wsl = false; let mut requires_case_sensitive_fs = false; + let mut requires_dir_permissions = false; let mut requires_engine = None; let mut requires_os = None; let mut serve_timeout_secs = None; @@ -205,6 +216,8 @@ impl ScenarioDecl { requires_wsl = true; } else if tag == REQUIRES_CASE_SENSITIVE_FS_TAG { requires_case_sensitive_fs = true; + } else if tag == REQUIRES_DIR_PERMISSIONS_TAG { + requires_dir_permissions = true; } else if tag == NIGHTLY_TAG { nightly = true; } else if tag == LIFECYCLE_TAG { @@ -222,6 +235,7 @@ impl ScenarioDecl { requires_bare_metal, requires_wsl, requires_case_sensitive_fs, + requires_dir_permissions, requires_engine, requires_os, serve_timeout_secs, @@ -577,6 +591,13 @@ pub fn resolve( .to_owned(), }; } + if decl.requires_dir_permissions && !cap.dir_permissions_bind { + return Expectation::Skip { + reason: "requires directory permissions that bind the user running the suite; a \ + read-only directory still takes new files here (root?)" + .to_owned(), + }; + } let engine = decl.effective_engine(cap); // A scenario that pins or defaults to a real engine and would actually serve // needs that engine to be startable here. We treat any GPU scenario with a @@ -685,6 +706,7 @@ mod tests { effective_serve_engine: "vllm".into(), platform_slug: "mi300x".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, "strix-ubuntu" => HostCapability { os_family: "linux".into(), @@ -696,6 +718,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, "strix-windows" => HostCapability { os_family: "windows".into(), @@ -707,6 +730,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo".into(), case_sensitive_fs: false, + dir_permissions_bind: true, }, // A WSL2 dev box with the ROCm passthrough in place: `os_family` is // `linux` and a GPU is usable, so nothing but `is_wsl` distinguishes @@ -727,6 +751,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo-wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, // The same box without the passthrough: the gfx target is reported // by the Windows-side driver but ROCm cannot reach it, so the probe @@ -741,6 +766,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, // The hosted WSL runner before the ROCm passthrough is complete: // Windows still reports the gfx target, but the CLI cannot use it. @@ -754,6 +780,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "strix-halo-wsl".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, _ => HostCapability { os_family: "other".into(), @@ -765,6 +792,7 @@ mod tests { effective_serve_engine: "lemonade".into(), platform_slug: "mock".into(), case_sensitive_fs: true, + dir_permissions_bind: true, }, } } @@ -1529,6 +1557,7 @@ serve_timeout_secs = 90 // so `@requires-os:linux` alone would have let the scenario run. let folded = HostCapability { case_sensitive_fs: false, + dir_permissions_bind: true, ..cap("wsl2") }; let both = decl(&[ @@ -1553,6 +1582,50 @@ serve_timeout_secs = 90 ); } + #[test] + fn dir_permissions_tag_parses_in_both_shapes() { + assert!(decl(&["id:x", "requires-dir-permissions"]).requires_dir_permissions); + assert!(decl(&["@id:x", "@requires-dir-permissions"]).requires_dir_permissions); + // Absent by default, so no existing scenario changes meaning. + assert!(!decl(&["id:x", "requires-os:linux"]).requires_dir_permissions); + } + + #[test] + fn requires_dir_permissions_skips_for_a_user_the_permission_bits_do_not_bind() { + let m = Expectations::default(); + let none = Included::default(); + let d = decl(&[ + "id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced", + "requires-os:linux", + "requires-dir-permissions", + ]); + assert_eq!( + resolve(&d, &cap("mi300x"), &m, none), + Expectation::ExpectPass, + "an ordinary user's read-only directory refuses new files, so the scenario runs" + ); + // The case the gate exists for: the WSL2 guest runs the suite as root, + // so `os_family` is `linux` and `@requires-os:linux` alone would let the + // scenario run and pass without testing anything. + let as_root = HostCapability { + dir_permissions_bind: false, + ..cap("wsl2") + }; + match resolve(&d, &as_root, &m, none) { + Expectation::Skip { reason } => assert!( + reason.contains("directory permissions"), + "the skip must name the missing premise: {reason}" + ), + other => panic!("a user the permission bits do not bind must skip, got {other:?}"), + } + // Without the tag the same host still runs it: the probe answer only + // matters to scenarios that declare the premise. + assert_eq!( + resolve(&decl(&["id:x", "requires-os:linux"]), &as_root, &m, none), + Expectation::ExpectPass + ); + } + #[test] fn os_reason_wins_over_case_folding_reason_on_a_non_matching_os() { // Native Windows fails both premises of the case-twin scenario; the grid @@ -1574,6 +1647,38 @@ serve_timeout_secs = 90 } } + #[test] + fn os_reason_wins_over_dir_permissions_reason_on_a_non_matching_os() { + // The same ordering as the case-folding twin above, for the tag the + // torn-audit-tail scenario carries next to `@requires-os:linux`. + let m = Expectations::default(); + let d = decl(&[ + "id:automations-torn-audit-tail-repaired-when-log-cannot-be-replaced", + "requires-os:linux", + "requires-dir-permissions", + ]); + assert!( + d.requires_dir_permissions && d.requires_os.is_some(), + "both premise tags must parse, or the ordering below proves nothing" + ); + let host = HostCapability { + dir_permissions_bind: false, + ..cap("strix-windows") + }; + assert_ne!(host.os_family, "linux", "fixture must fail the OS premise"); + assert!( + !host.dir_permissions_bind, + "fixture must fail both premises" + ); + match resolve(&d, &host, &m, Included::default()) { + Expectation::Skip { reason } => assert!( + reason.contains("requires os 'linux'"), + "the OS reason must win: {reason}" + ), + other => panic!("native Windows must skip, got {other:?}"), + } + } + #[test] fn case_folding_skip_is_not_recorded_as_a_known_bug() { // A host that cannot hold the premise is not-applicable, not a defect, so @@ -1593,6 +1698,7 @@ reason = "unrelated open bug" ]); let folded = HostCapability { case_sensitive_fs: false, + dir_permissions_bind: true, ..cap("strix-ubuntu") }; assert!(matches!( diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 0bf3b82544..8e45c6f041 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -690,6 +690,12 @@ impl Drop for E2eWorld { if let Some(root) = &self.isolated_root { stop_managed_services(root.path()); } + // A scenario that made the audit directory read-only must give it back + // before the TempDir below tries to remove it, even if a step panicked. + #[cfg(unix)] + if self.isolated_root.is_some() { + e2e::automations_steps::restore_audit_dir_permissions(&self.app_paths().audit_dir()); + } // `isolated_root` is a `TempDir`; its own Drop removes the directory. } } diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 3a3534860d..0204fe8bc4 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -14,7 +14,8 @@ //! guard skips the spawn. Contracts verified against the running Linux binary //! (EAI-8072). Scoped to the enable/disable/mode slice, the proposal-history //! listing's unit formatting, the bare listing's tolerance of a torn -//! audit-log tail, and the size cap on the audit log and compaction of the +//! audit-log tail, the repair of one by the next audited command, and the +//! size cap on the audit log and compaction of the //! proposal history (ROCMAI-729); the watcher/daemon trigger behaviour itself //! is covered separately. @@ -22,7 +23,7 @@ use cucumber::{given, then, when}; use rocm_core::{ AuditEventRecord, AutomationProposalRecord, EVENT_LOG_ROTATE_BYTES, EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, append_automation_proposal, - load_automation_proposals, + jsonl_lock_path, load_automation_proposals, }; use crate::E2eWorld; @@ -181,6 +182,95 @@ async fn torn_audit_tail(world: &mut E2eWorld) { .expect("failed to seed a torn audit-event tail"); } +/// Make the audit log's directory refuse new files while the log inside it +/// stays writable, so the repair of a torn tail cannot write the clean copy it +/// renames into place — the failure a full disk causes, without filling one. +/// +/// The CLI's lock file for the log is created first: it is reused when it +/// exists but cannot be created in a directory that takes no new files. Fails +/// if the directory still takes new files afterwards: directory permissions do +/// not bind root, and the scenario's `@requires-dir-permissions` tag skips it +/// there, so this is the backstop that keeps it from passing without +/// exercising the fallback should the tag be dropped. +/// [`restore_audit_dir_permissions`] undoes it so the scenario's temp root can +/// be removed. +#[given("the audit log's directory takes no new files")] +async fn audit_dir_takes_no_new_files(world: &mut E2eWorld) { + let paths = world.app_paths(); + let dir = paths.audit_dir(); + std::fs::File::create(jsonl_lock_path(&paths.audit_events_path())) + .expect("failed to create the lock file"); + forbid_new_files(&dir); + let probe = dir.join("probe"); + if std::fs::File::create(&probe).is_ok() { + let _ = std::fs::remove_file(&probe); + panic!( + "{} still takes new files, so this scenario cannot make the audit log's \ + replacement fail: directory permissions do not bind this user (root?)", + dir.display() + ); + } +} + +#[cfg(unix)] +fn forbid_new_files(dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o555)) + .expect("failed to make the audit directory read-only"); +} + +#[cfg(not(unix))] +// Platform stub for a `@requires-os:linux` scenario. The Unix arm does I/O and +// cannot be `const`, so making only this arm `const` would give the two +// platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn forbid_new_files(_dir: &std::path::Path) { + panic!("this step needs Unix directory permissions; its scenario is @requires-os:linux"); +} + +/// Undo [`audit_dir_takes_no_new_files`] on `dir`, best effort, from the +/// world's teardown. +#[cfg(unix)] +pub(crate) fn restore_audit_dir_permissions(dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + if dir.is_dir() { + let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o755)); + } +} + +#[then("the CLI prints no warning about the audit log")] +async fn no_audit_warning(world: &mut E2eWorld) { + let stderr = world.cli_stderr.clone().unwrap_or_default(); + assert!( + !stderr.to_lowercase().contains("audit"), + "expected no audit-log warning, got stderr:\n{stderr}" + ); +} + +#[then( + "the audit log keeps its earlier record, drops the torn line and records the permissions change" +)] +async fn audit_log_repaired_and_appended(world: &mut E2eWorld) { + let path = world.app_paths().audit_events_path(); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + // `audit_records` parses every line, so a left-over torn fragment fails here. + let records = audit_records(&path); + let seen: Vec<(&str, &str)> = records + .iter() + .map(|record| (record.message.as_str(), record.action.as_str())) + .collect(); + assert_eq!( + seen, + [ + ("clean", "append"), + ("permissions mode set to ask", "set_mode") + ], + "unexpected audit log:\n{text}" + ); + assert!(text.ends_with('\n'), "the last record must be terminated"); +} + #[given("an enabled automation watcher")] async fn enabled_watcher(world: &mut E2eWorld) { suppress_daemon_spawn(world);