From 9f4731f69f8d2146694b67d6bd2e2f6b5c40681e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 08:49:05 +0000 Subject: [PATCH 1/8] ROCMAI-721: stop JSONL stores losing records to concurrent writers The automation events, audit and proposals stores were each updated by reading the whole file, changing it in memory and renaming a rewritten copy over the original, with no lock. The rocm CLI and rocmd write these stores concurrently, so two overlapping writers kept only the last rename and the other record was dropped with no error. Two threads of one process could also collide on the same temp file name. Every mutation of a store now holds a sibling `.lock` file (rocm_core::FileLock: flock on Linux, LockFileEx on Windows): - Appending a record is a single append-mode write of one line under that lock. It no longer reads or rewrites the file, so its cost no longer grows with the store (ROCMAI-665). A torn last line left by a killed writer, or by a write that failed partway, is cut by the next append (see the next commit for how). - A proposal status update keeps its whole-file rewrite, but its full read-modify-write now runs under the same lock, so it can neither drop a concurrently appended proposal nor strand one in a replaced file. Readers stay lock-free. An unterminated last line that is not valid JSON (an append caught mid-write) is skipped instead of failing the read, and the proposals reader works on bytes so a cut inside a UTF-8 character cannot fail the whole file. load_recent_jsonl_records now scans backwards from the end and stops after `limit` records instead of parsing the whole store (ROCMAI-665). Regression tests reproduce the loss first: N threads and N separate processes (the test binary re-executed) appending to all three stores, and status updates racing appends. All three failed before this change (e.g. 86 of 120 audit records survived six writer processes) and pass after it. Size caps/rotation for the stores and tail-only proposal reads are left for a follow-up. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 9 +- crates/rocm-core/src/automation.rs | 900 +++++++++++++++++++++++++++-- crates/rocm-core/src/lib.rs | 4 + 3 files changed, 864 insertions(+), 49 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 0be3f3c31f..115cbb0b2e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4566,9 +4566,12 @@ fn apply_service_prune_plan( /// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so /// a second acquire on a path already held by this process would hang forever. /// Nothing under either phase acquires this path: the only other -/// `FileLock::acquire` calls in the tree are `ensure_background_helper_running_quiet` -/// and the managed-uv bootstrap in `rocm_core::uv`, each on a different lock file, -/// and neither is reachable from here. In the other +/// `FileLock::acquire` calls in the tree are `ensure_background_helper_running_quiet`, +/// the managed-uv bootstrap in `rocm_core::uv`, and the per-store locks +/// `rocm_core`'s automation module takes around each audit, event and proposal +/// write — each on a different lock file — and none is reachable from here: +/// `rocm services prune` records its audit event only after this returns and +/// the lock is released. In the other /// direction `serve` holds this lock but never runs prune, in-process or as a /// subprocess. fn prune_managed_service_records( diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index b8264c23a0..d2b8d1a17f 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -8,21 +8,30 @@ //! the running daemon's live snapshot (`AutomationRuntimeState`), the //! logically append-only JSONL stores for watcher action events //! (`AutomationEventRecord`), audit log entries, and review proposals, and -//! the accessors used to read them back. Every write to the JSONL stores — -//! an appended record or a full rewrite on proposal status update — goes -//! through `write_atomic`, so a crash or full disk mid-write never leaves a -//! truncated store. `AutomationTriggerEvent` is also defined here, but this +//! the accessors used to read them back. +//! +//! The `rocm` CLI and `rocmd` write these stores concurrently, so every +//! mutation of a store holds its sibling `.lock` file (see `jsonl_lock_path`). +//! Appending a record is one append-mode write of one line — its cost does not +//! grow with the store — rolled back if the write fails; a proposal status +//! update rewrites the store via `write_atomic`. Readers take no lock, and an +//! unterminated last line that is not valid JSON — an append caught mid-write — +//! is skipped rather than reported as corruption. The stores are not rotated +//! or capped. `AutomationTriggerEvent` is also defined here, but this //! module persists none of it: it is the in-memory trigger type passed //! between `rocmd`'s watcher and webhook plumbing (`apps/rocmd/src/watchers.rs`, //! `apps/rocmd/src/webhook.rs`), never serialized. use anyhow::{Context, Result, bail}; +use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::fs; -use std::path::Path; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use crate::AppPaths; +use crate::FileLock; use crate::WatcherMode; use crate::require_nonempty; use crate::unix_time_millis; @@ -207,50 +216,310 @@ pub struct AuditEventRecord { pub service_id: Option, } -/// Append one JSONL `record` to `path`: read the existing content (if any), -/// append the serialized record plus a newline, and replace the file via -/// `write_atomic`. +/// Lock file serializing every mutation of the JSONL store at `path` — a +/// sibling named `.lock` (e.g. `events.jsonl.lock`). +/// +/// Appends and the proposal-status rewrite both take it. The rewrite needs it +/// for its read-modify-write; appends need it too, because the rewrite +/// replaces the store by renaming a new file over it, and an append that had +/// already opened the old file would land in the replaced copy and vanish. +fn jsonl_lock_path(path: &Path) -> PathBuf { + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(".lock"); + path.with_file_name(name) +} + +/// How many bytes the backward scans below read per step. +const JSONL_SCAN_CHUNK: u64 = 64 * 1024; + +/// Append one JSONL `record` to `path`, creating the file if needed. +/// +/// Holds the store's [`jsonl_lock_path`] lock and writes the line with a +/// single append-mode write, so the cost no longer grows with the store and +/// a concurrent writer can never drop this record (or have its own record +/// dropped by this one), across threads and processes alike. +/// +/// A failed write is rolled back to the pre-append length, so a full disk +/// never leaves a torn line behind. A torn line left by a writer that was +/// killed mid-write is cleaned up by the next append: see +/// [`repair_unterminated_tail`]. fn append_jsonl_record(path: &Path, record: &T, context_label: &str) -> Result<()> { - let mut line = serde_json::to_string(record) + let mut line = serde_json::to_vec(record) .with_context(|| format!("failed to serialize {context_label}"))?; - line.push('\n'); - let mut existing = if path.is_file() { - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))? - } else { - String::new() - }; - existing.push_str(&line); - crate::write_atomic(path, existing.as_bytes(), "jsonl") + line.push(b'\n'); + + let _lock = FileLock::acquire(jsonl_lock_path(path))?; + let file = fs::OpenOptions::new() + .read(true) + .append(true) + .create(true) + .open(path) + .with_context(|| format!("failed to open {}", path.display()))?; + let (start, needs_separator) = repair_unterminated_tail(path, &file)?; + if needs_separator { + line.insert(0, b'\n'); + } + if let Err(error) = (&file).write_all(&line) { + // Best effort: if this fails too, the torn tail is still repaired by + // the next append, and readers already skip it. + let _ = truncate_file(path, start); + return Err(error).with_context(|| format!("failed to append to {}", path.display())); + } + Ok(()) +} + +/// Make the store at `path` end on a line boundary before an append, returning +/// the length to roll back to if the append fails and whether the new line +/// needs a leading `\n`. +/// +/// Must be called with the store's lock held, so no other writer is mid-write +/// and an unterminated last line is either crash debris (a writer killed +/// mid-write: not valid JSON, cut off) or a complete record whose newline was +/// lost to a hand edit (valid JSON, kept and terminated). +fn repair_unterminated_tail(path: &Path, file: &fs::File) -> Result<(u64, bool)> { + let len = file + .metadata() + .with_context(|| format!("failed to stat {}", path.display()))? + .len(); + // The common case — empty, or ending on a line boundary — costs one byte. + if len == 0 || read_range(path, file, len - 1, len)? == b"\n" { + return Ok((len, false)); + } + let mut tail = Vec::new(); + let mut tail_start = len; + while tail_start > 0 { + let chunk_start = tail_start.saturating_sub(JSONL_SCAN_CHUNK); + let chunk = read_range(path, file, chunk_start, tail_start)?; + if let Some(newline) = chunk.iter().rposition(|&byte| byte == b'\n') { + tail.splice(0..0, chunk[newline + 1..].iter().copied()); + tail_start = chunk_start + newline as u64 + 1; + break; + } + tail.splice(0..0, chunk); + tail_start = chunk_start; + } + if tail.is_empty() { + return Ok((len, false)); + } + if is_blank(&tail) || !is_torn_record(&tail) { + return Ok((len, true)); + } + truncate_file(path, tail_start)?; + Ok((tail_start, false)) +} + +/// Read bytes `[start, end)` of `file`. +/// +/// Returns fewer bytes if the file now ends before `end`: a lock-free reader +/// can race a writer cutting a torn last line (see +/// [`repair_unterminated_tail`]), and the bytes cut are never a complete +/// record, so what is still there is all the caller needs. +fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result> { + let mut reader = file; + reader + .seek(SeekFrom::Start(start)) + .with_context(|| format!("failed to seek in {}", path.display()))?; + let mut bytes = Vec::new(); + reader + .take(end - start) + .read_to_end(&mut bytes) + .with_context(|| format!("failed to read {}", path.display()))?; + Ok(bytes) +} + +/// Cut `path` back to `len` bytes. Opens its own write handle: the append +/// handle cannot be relied on to truncate on Windows, where append-only +/// access does not include the right to move end-of-file backwards. +fn truncate_file(path: &Path, len: u64) -> Result<()> { + fs::OpenOptions::new() + .write(true) + .open(path) + .and_then(|file| file.set_len(len)) + .with_context(|| format!("failed to truncate {}", path.display())) +} + +/// Parse a whole JSONL store, pairing each record with its line index. +/// +/// Readers take no lock, so one may catch an append mid-write: a last line +/// with no terminating `\n` that is not valid JSON is that in-flight (or +/// crash-torn) record, and is skipped rather than failing the whole read. Any +/// other unparseable line is still an error. +/// +/// Works on bytes, not a `String`: a record cut mid-write can end inside a +/// multi-byte UTF-8 character, which would fail a whole-file UTF-8 decode +/// before the skip could apply. +fn parse_jsonl_records( + bytes: &[u8], + path: &Path, + context_label: &str, +) -> Result> { + let unterminated = !bytes.is_empty() && !bytes.ends_with(b"\n"); + let line_count = bytes.split(|&byte| byte == b'\n').count(); + let mut records = Vec::new(); + for (index, line) in bytes.split(|&byte| byte == b'\n').enumerate() { + if is_blank(line) { + continue; + } + match serde_json::from_slice::(line) { + Ok(record) => records.push((index, record)), + Err(_) if unterminated && index + 1 == line_count && is_torn_record(line) => {} + Err(error) => { + return Err(error).with_context(|| { + format!("failed to parse {context_label} in {}", path.display()) + }); + } + } + } + Ok(records) } -/// Read every JSONL record from `path` and keep only the most recent `limit` -/// (in on-disk order). Returns an empty vec when `path` doesn't exist yet or -/// `limit` is `0`. -fn load_recent_jsonl_records( +/// Read the most recent `limit` JSONL records from `path`, in on-disk order. +/// Returns an empty vec when `path` doesn't exist yet or `limit` is `0`. +/// +/// Scans backwards from the end of the file and stops once it has `limit` +/// records, so the cost tracks `limit`, not the size of the store; a corrupt +/// line older than those records is therefore not reported. As in +/// [`parse_jsonl_records`], an unterminated last line that is not valid JSON is +/// an append caught mid-write and is skipped. +/// +/// The scan takes no lock, so a writer may cut a torn last line (see +/// [`repair_unterminated_tail`]) and append again while it runs. A scan that +/// fails while the file's length moved under it is retried from the new end, +/// so that race never surfaces as an error. +fn load_recent_jsonl_records( path: &Path, limit: usize, context_label: &str, ) -> Result> { + const ATTEMPTS: usize = 3; if !path.is_file() || limit == 0 { return Ok(Vec::new()); } - let text = - fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; - let mut records = Vec::new(); - for line in text.lines() { - if line.trim().is_empty() { - continue; + let file = + fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; + let file_len = || { + file.metadata() + .map(|metadata| metadata.len()) + .with_context(|| format!("failed to stat {}", path.display())) + }; + let mut attempt = 1; + loop { + let len = file_len()?; + match scan_recent_jsonl_records(path, &file, len, limit, context_label) { + Err(_) if attempt < ATTEMPTS && file_len()? != len => attempt += 1, + outcome => return outcome, } - let record = serde_json::from_str::(line) - .with_context(|| format!("failed to parse {context_label} in {}", path.display()))?; - records.push(record); } - if records.len() > limit { - records.drain(0..records.len() - limit); +} + +/// One backward scan of [`load_recent_jsonl_records`], over the first `len` +/// bytes of `file`. +fn scan_recent_jsonl_records( + path: &Path, + file: &fs::File, + len: u64, + limit: usize, + context_label: &str, +) -> Result> { + let mut lines = ReverseLines::new(path, file, len, JSONL_SCAN_CHUNK); + let parse = |line: &[u8]| { + serde_json::from_slice::(line) + .with_context(|| format!("failed to parse {context_label} in {}", path.display())) + }; + let mut records = Vec::new(); + // The first segment is whatever follows the last `\n`: empty when the + // file ends on a line boundary, else an unterminated last line. + if let Some(tail) = lines.next_line()? + && !is_blank(&tail) + && !is_torn_record(&tail) + { + records.push(parse(&tail)?); + } + while records.len() < limit { + let Some(line) = lines.next_line()? else { + break; + }; + if !is_blank(&line) { + records.push(parse(&line)?); + } } + records.truncate(limit); + records.reverse(); Ok(records) } +fn is_blank(line: &[u8]) -> bool { + line.iter().all(u8::is_ascii_whitespace) +} + +/// Whether an unterminated last line is a record cut off mid-write, as +/// opposed to a complete one that merely lost its `\n` (e.g. to a hand edit). +/// +/// The test is "not valid JSON", not "not a valid record": a cut-off JSON +/// object never parses, while complete JSON of the wrong shape is real data — +/// readers report it as before, and writers keep it rather than cut it. +fn is_torn_record(line: &[u8]) -> bool { + serde_json::from_slice::(line).is_err() +} + +/// Yields the `\n`-separated segments of a file from last to first, reading +/// it backwards in fixed-size chunks. The first segment yielded is the one +/// after the final `\n` (empty when the file ends with one). +struct ReverseLines<'a> { + path: &'a Path, + file: &'a fs::File, + chunk: u64, + /// File offset of `pending[0]`; everything before it is still unread. + pos: u64, + /// Bytes read but not yet yielded, i.e. file bytes `[pos, pos + len)`. + pending: Vec, + done: bool, +} + +impl<'a> ReverseLines<'a> { + /// Read the first `len` bytes of `file` backwards, `chunk` bytes a step. + const fn new(path: &'a Path, file: &'a fs::File, len: u64, chunk: u64) -> Self { + Self { + path, + file, + chunk: if chunk == 0 { 1 } else { chunk }, + pos: len, + pending: Vec::new(), + done: false, + } + } + + fn next_line(&mut self) -> Result>> { + if self.done { + return Ok(None); + } + loop { + // Scans from the end, so it stops at the nearest `\n`: a line is + // rescanned only when it is longer than one chunk. + if let Some(newline) = self.pending.iter().rposition(|&byte| byte == b'\n') { + let line = self.pending.split_off(newline + 1); + self.pending.truncate(newline); + return Ok(Some(line)); + } + if self.pos == 0 { + self.done = true; + return Ok(Some(std::mem::take(&mut self.pending))); + } + let start = self.pos.saturating_sub(self.chunk); + let mut bytes = read_range(self.path, self.file, start, self.pos)?; + // A short first read just means a torn tail was cut (see + // `read_range`). Short after that, the bytes already read no longer + // continue from these, so splicing them would invent a line. + if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { + bail!("{} shrank while it was being read", self.path.display()); + } + bytes.append(&mut self.pending); + self.pending = bytes; + self.pos = start; + } + } +} + pub fn append_automation_event(paths: &AppPaths, event: &AutomationEventRecord) -> Result<()> { paths.ensure()?; append_jsonl_record( @@ -286,19 +555,16 @@ pub fn load_automation_proposals(paths: &AppPaths) -> Result(line) - .with_context(|| format!("failed to parse proposal record in {}", path.display()))?; - normalize_proposal_identity(&mut proposal, index); - proposals.push(proposal); - } - Ok(proposals) + let bytes = fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + Ok( + parse_jsonl_records::(&bytes, &path, "proposal record")? + .into_iter() + .map(|(index, mut proposal)| { + normalize_proposal_identity(&mut proposal, index); + proposal + }) + .collect(), + ) } pub fn load_recent_automation_proposals( @@ -324,6 +590,11 @@ pub fn find_automation_proposal( .with_context(|| format!("automation proposal `{proposal_id}` not found")) } +/// Set one proposal's `status` by rewriting the proposals store. +/// +/// The whole read-modify-write runs under the store's [`jsonl_lock_path`] +/// lock, so a proposal appended (or another status set) meanwhile is neither +/// dropped nor overwritten. pub fn update_automation_proposal_status( paths: &AppPaths, proposal_id: &str, @@ -331,6 +602,8 @@ pub fn update_automation_proposal_status( ) -> Result { require_nonempty(proposal_id, "proposal_id")?; require_nonempty(status, "status")?; + paths.ensure()?; + let _lock = FileLock::acquire(jsonl_lock_path(&paths.automation_proposals_path()))?; let mut proposals = load_automation_proposals(paths)?; let Some(proposal) = proposals .iter_mut() @@ -351,11 +624,12 @@ pub fn load_recent_audit_events(paths: &AppPaths, limit: usize) -> Result Result<()> { - paths.ensure()?; let path = paths.automation_proposals_path(); let mut text = String::new(); for proposal in proposals { @@ -592,4 +866,538 @@ mod tests { assert_eq!(events[0].action, "proposal_rejected"); Ok(()) } + + fn audit_record(tag: &str) -> AuditEventRecord { + AuditEventRecord { + at_unix_ms: 1, + source: "test".to_owned(), + category: "concurrency".to_owned(), + actor: "writer".to_owned(), + level: "info".to_owned(), + action: "append".to_owned(), + message: tag.to_owned(), + watcher_id: None, + service_id: None, + } + } + + fn event_record(tag: &str) -> AutomationEventRecord { + AutomationEventRecord { + at_unix_ms: 1, + watcher_id: "gpu-metrics".to_owned(), + level: "info".to_owned(), + action: "record_gpu_metrics".to_owned(), + message: tag.to_owned(), + service_id: None, + } + } + + fn proposal_record(tag: &str) -> AutomationProposalRecord { + AutomationProposalRecord { + at_unix_ms: 1, + proposal_id: tag.to_owned(), + watcher_id: "cache-warm".to_owned(), + action: "queue_prefetch_proposal".to_owned(), + title: "Prefetch".to_owned(), + message: tag.to_owned(), + status: "pending".to_owned(), + service_id: None, + tool: None, + arguments: serde_json::json!({}), + reviewed_at_unix_ms: None, + } + } + + /// Append one record tagged `tag` to each of the three JSONL stores. + fn append_to_every_store(paths: &AppPaths, tag: &str) -> Result<()> { + append_audit_event(paths, &audit_record(tag))?; + append_automation_event(paths, &event_record(tag))?; + append_automation_proposal(paths, &proposal_record(tag)) + } + + /// The tags in each store, in on-disk order, read straight from the files + /// so a lost record cannot hide behind any loader's filtering. + fn tags_in_every_store(paths: &AppPaths) -> Result<[Vec; 3]> { + let audit = fs::read_to_string(paths.audit_events_path())? + .lines() + .map(|line| serde_json::from_str::(line).map(|r| r.message)) + .collect::, _>>()?; + let events = fs::read_to_string(paths.automation_events_path())? + .lines() + .map(|line| serde_json::from_str::(line).map(|r| r.message)) + .collect::, _>>()?; + let proposals = fs::read_to_string(paths.automation_proposals_path())? + .lines() + .map(|line| { + serde_json::from_str::(line).map(|r| r.proposal_id) + }) + .collect::, _>>()?; + Ok([audit, events, proposals]) + } + + fn assert_every_store_holds_exactly(stores: [Vec; 3], expected: &[String]) { + let mut expected = expected.to_vec(); + expected.sort(); + for (store, mut tags) in ["audit", "events", "proposals"].into_iter().zip(stores) { + tags.sort(); + assert_eq!( + tags.len(), + expected.len(), + "{store} store lost or duplicated records under concurrent writers" + ); + assert_eq!( + tags, expected, + "{store} store holds the wrong set of records" + ); + } + } + + // Regression test for concurrent writers silently dropping each other's + // records: every append used to read the whole store, add one line in + // memory, and rename a rewritten copy over the original, so two writers + // that overlapped kept only the last rename (and two threads of one + // process even shared a temp-file name). + #[test] + fn concurrent_appends_from_many_threads_lose_no_records() -> Result<()> { + const WRITERS: usize = 8; + const APPENDS_PER_WRITER: usize = 25; + let (root, paths) = temp_app_paths("jsonl-concurrent-threads"); + paths.ensure()?; + let barrier = std::sync::Barrier::new(WRITERS); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..WRITERS) + .map(|writer| { + let (paths, barrier) = (&paths, &barrier); + scope.spawn(move || -> Result<()> { + barrier.wait(); + for n in 0..APPENDS_PER_WRITER { + append_to_every_store(paths, &format!("t{writer}-{n}"))?; + } + Ok(()) + }) + }) + .collect(); + for handle in handles { + handle.join().expect("writer thread panicked")?; + } + Ok::<_, anyhow::Error>(()) + })?; + + let expected: Vec = (0..WRITERS) + .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("t{writer}-{n}"))) + .collect(); + let stores = tags_in_every_store(&paths); + fs::remove_dir_all(root).ok(); + assert_every_store_holds_exactly(stores?, &expected); + Ok(()) + } + + /// Env var that turns a re-exec of this test binary into one writer of + /// [`concurrent_appends_from_many_processes_lose_no_records`]: its value is + /// `|`. + const CHILD_WRITER_ENV: &str = "ROCM_CORE_TEST_JSONL_CHILD_WRITER"; + const PROCESS_APPENDS_PER_WRITER: usize = 20; + + // Same regression as the thread test, across processes — the shape the + // stores actually see (the `rocm` CLI and `rocmd` both write the audit + // store, and both touch the proposals store). + #[test] + fn concurrent_appends_from_many_processes_lose_no_records() -> Result<()> { + const WRITERS: usize = 6; + if let Some(spec) = std::env::var_os(CHILD_WRITER_ENV) { + let spec = spec.to_string_lossy().into_owned(); + let (data_dir, writer) = spec.split_once('|').context("malformed writer spec")?; + let paths = AppPaths { + config_dir: Path::new(data_dir).join("config"), + data_dir: Path::new(data_dir).to_path_buf(), + cache_dir: Path::new(data_dir).join("cache"), + }; + for n in 0..PROCESS_APPENDS_PER_WRITER { + append_to_every_store(&paths, &format!("p{writer}-{n}"))?; + } + return Ok(()); + } + + let (root, paths) = temp_app_paths("jsonl-concurrent-processes"); + paths.ensure()?; + let exe = std::env::current_exe().context("current test executable")?; + let children = (0..WRITERS) + .map(|writer| { + std::process::Command::new(&exe) + .args([ + "automation::tests::concurrent_appends_from_many_processes_lose_no_records", + "--exact", + "--quiet", + ]) + .env( + CHILD_WRITER_ENV, + format!("{}|{writer}", paths.data_dir.display()), + ) + .stdout(std::process::Stdio::null()) + .spawn() + .context("spawn writer process") + }) + .collect::>>()?; + for mut child in children { + let status = child.wait()?; + assert!(status.success(), "writer process failed: {status:?}"); + } + + let expected: Vec = (0..WRITERS) + .flat_map(|writer| { + (0..PROCESS_APPENDS_PER_WRITER).map(move |n| format!("p{writer}-{n}")) + }) + .collect(); + let stores = tags_in_every_store(&paths); + fs::remove_dir_all(root).ok(); + assert_every_store_holds_exactly(stores?, &expected); + Ok(()) + } + + // A proposal-status update rewrites the whole proposals store, so it must + // neither drop proposals appended while it runs nor be undone by them. + #[test] + fn proposal_status_updates_racing_appends_lose_neither() -> Result<()> { + const APPENDS: usize = 100; + const UPDATES: usize = 50; + let (root, paths) = temp_app_paths("proposal-status-race"); + append_automation_proposal(&paths, &proposal_record("reviewed"))?; + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|scope| { + let appender = scope.spawn(|| -> Result<()> { + barrier.wait(); + for n in 0..APPENDS { + append_automation_proposal(&paths, &proposal_record(&format!("new-{n}")))?; + } + Ok(()) + }); + let reviewer = scope.spawn(|| -> Result<()> { + barrier.wait(); + for n in 0..UPDATES { + let status = if n % 2 == 0 { "approved" } else { "rejected" }; + update_automation_proposal_status(&paths, "reviewed", status)?; + } + Ok(()) + }); + appender.join().expect("appender panicked")?; + reviewer.join().expect("reviewer panicked")?; + Ok::<_, anyhow::Error>(()) + })?; + + let proposals = load_automation_proposals(&paths)?; + fs::remove_dir_all(root).ok(); + let mut ids: Vec<&str> = proposals.iter().map(|p| p.proposal_id.as_str()).collect(); + ids.sort_unstable(); + let mut expected: Vec = (0..APPENDS).map(|n| format!("new-{n}")).collect(); + expected.push("reviewed".to_owned()); + expected.sort(); + assert_eq!( + ids, expected, + "a status rewrite dropped an appended proposal" + ); + let reviewed = proposals + .iter() + .find(|p| p.proposal_id == "reviewed") + .expect("reviewed proposal present"); + // UPDATES is even, so the last update (n = UPDATES - 1) is odd: rejected. + assert_eq!( + reviewed.status, "rejected", + "the last status update was lost" + ); + Ok(()) + } + + // Readers take no lock, so they can catch an append mid-write; that must + // never surface as a parse error from `rocm status` or the dashboard. + // This is a smoke test: on Linux a small append is rarely seen half-done, + // so the skip itself is pinned deterministically by + // `readers_skip_an_unterminated_last_line_that_does_not_parse`. + #[test] + fn reads_racing_appends_never_fail() -> Result<()> { + const APPENDS: usize = 200; + let (root, paths) = temp_app_paths("jsonl-read-race"); + paths.ensure()?; + let done = std::sync::atomic::AtomicBool::new(false); + let outcome = std::thread::scope(|scope| { + let reader = scope.spawn(|| -> Result { + let mut reads = 0; + while !done.load(std::sync::atomic::Ordering::Acquire) { + load_recent_audit_events(&paths, 5)?; + load_recent_automation_events(&paths, 5)?; + load_automation_proposals(&paths)?; + reads += 1; + } + Ok(reads) + }); + let written = + (0..APPENDS).try_for_each(|n| append_to_every_store(&paths, &format!("r{n}"))); + done.store(true, std::sync::atomic::Ordering::Release); + let reads = reader.join().expect("reader panicked"); + written.and(reads) + }); + let loaded = load_recent_audit_events(&paths, APPENDS); + fs::remove_dir_all(root).ok(); + outcome?; + assert_eq!(loaded?.len(), APPENDS); + Ok(()) + } + + /// Write `contents` as the audit store, creating its directory. + fn seed_audit_store(paths: &AppPaths, contents: &str) -> Result<()> { + paths.ensure()?; + fs::write(paths.audit_events_path(), contents)?; + Ok(()) + } + + fn audit_line(tag: &str) -> String { + serde_json::to_string(&audit_record(tag)).expect("serialize audit record") + } + + fn audit_messages(records: &[AuditEventRecord]) -> Vec<&str> { + records.iter().map(|r| r.message.as_str()).collect() + } + + #[test] + fn reverse_lines_matches_a_forward_split_at_every_chunk_size() -> Result<()> { + let (root, _paths) = temp_app_paths("reverse-lines"); + fs::create_dir_all(&root)?; + let path = root.join("lines.jsonl"); + for contents in [ + "", + "\n", + "a", + "a\n", + "a\nbb\n", + "a\nbb\nccc", + "\n\nx\n\n", + "long-line-crossing-chunks\nshort\n", + ] { + fs::write(&path, contents)?; + let file = fs::File::open(&path)?; + let mut expected: Vec<&[u8]> = contents.as_bytes().split(|&b| b == b'\n').collect(); + expected.reverse(); + for chunk in 1..=8 { + let mut lines = ReverseLines::new(&path, &file, file.metadata()?.len(), chunk); + let mut got = Vec::new(); + while let Some(line) = lines.next_line()? { + got.push(line); + } + assert_eq!(got, expected, "contents {contents:?}, chunk {chunk}"); + } + } + fs::remove_dir_all(root).ok(); + Ok(()) + } + + #[test] + fn load_recent_stops_before_corrupt_lines_older_than_the_window() -> Result<()> { + let (root, paths) = temp_app_paths("recent-window"); + seed_audit_store( + &paths, + &format!( + "not json\n{}\n\n{}\n{}\n", + audit_line("a"), + audit_line("b"), + audit_line("c") + ), + )?; + let recent = load_recent_audit_events(&paths, 3); + let too_far = load_recent_audit_events(&paths, 4); + fs::remove_dir_all(root).ok(); + assert_eq!(audit_messages(&recent?), ["a", "b", "c"]); + // Reaching the corrupt line is still an error, as before. + assert!( + too_far.is_err(), + "a corrupt line inside the window must fail" + ); + Ok(()) + } + + #[test] + fn readers_skip_an_unterminated_last_line_that_does_not_parse() -> Result<()> { + let (root, paths) = temp_app_paths("unterminated-torn"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let torn_proposal = &serde_json::to_string(&proposal_record("torn"))?[..20]; + fs::write( + paths.automation_proposals_path(), + format!( + "{}\n{torn_proposal}", + serde_json::to_string(&proposal_record("a"))? + ), + )?; + let audit = load_recent_audit_events(&paths, 5); + let proposals = load_automation_proposals(&paths); + fs::remove_dir_all(root).ok(); + assert_eq!(audit_messages(&audit?), ["a"]); + let ids: Vec = proposals?.into_iter().map(|p| p.proposal_id).collect(); + assert_eq!(ids, ["a"]); + Ok(()) + } + + // `serde_json` writes non-ASCII text as raw UTF-8, so a record cut + // mid-write can end inside a character; that must not fail a whole-file + // UTF-8 decode and block every proposal read, approve and reject. + #[test] + fn proposal_reads_skip_a_torn_line_cut_inside_a_utf8_character() -> Result<()> { + let (root, paths) = temp_app_paths("unterminated-utf8"); + paths.ensure()?; + let mut torn = proposal_record("torn"); + torn.message = "ü".repeat(8); + let torn = serde_json::to_vec(&torn)?; + let cut = torn + .iter() + .position(|&byte| byte == 0xC3) + .expect("message is raw UTF-8") + + 1; + let mut contents = serde_json::to_vec(&proposal_record("a"))?; + contents.push(b'\n'); + contents.extend_from_slice(&torn[..cut]); + fs::write(paths.automation_proposals_path(), contents)?; + let proposals = load_automation_proposals(&paths); + let updated = update_automation_proposal_status(&paths, "a", "approved"); + fs::remove_dir_all(root).ok(); + let ids: Vec = proposals?.into_iter().map(|p| p.proposal_id).collect(); + assert_eq!(ids, ["a"]); + assert_eq!(updated?.status, "approved"); + Ok(()) + } + + // Complete JSON that is not a valid record is real data, not an append + // caught mid-write: reads still report it, and a status rewrite must fail + // rather than silently drop it from disk. + #[test] + fn an_unterminated_last_line_of_the_wrong_shape_is_reported_not_dropped() -> Result<()> { + let (root, paths) = temp_app_paths("unterminated-wrong-shape"); + paths.ensure()?; + let contents = format!( + "{}\n{{\"unexpected\":true}}", + serde_json::to_string(&proposal_record("a"))? + ); + fs::write(paths.automation_proposals_path(), &contents)?; + seed_audit_store( + &paths, + &format!("{}\n{{\"unexpected\":true}}", audit_line("a")), + )?; + let proposals = load_automation_proposals(&paths); + let updated = update_automation_proposal_status(&paths, "a", "approved"); + let on_disk = fs::read_to_string(paths.automation_proposals_path()); + let audit = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert!(proposals.is_err(), "a wrong-shape record must be reported"); + assert!( + updated.is_err(), + "the rewrite must refuse, not drop the line" + ); + assert_eq!(on_disk?, contents); + assert!(audit.is_err(), "a wrong-shape record must be reported"); + Ok(()) + } + + // A lock-free reader that read the length just before a writer cut a torn + // tail sees the file end early; that is not an error. + #[test] + fn a_scan_that_finds_the_file_shorter_than_it_was_still_succeeds() -> Result<()> { + let (root, paths) = temp_app_paths("scan-shrunk"); + seed_audit_store( + &paths, + &format!("{}\n{}\n", audit_line("a"), audit_line("b")), + )?; + let path = paths.audit_events_path(); + let file = fs::File::open(&path)?; + let stale_len = file.metadata()?.len() + 40; + let recent = + scan_recent_jsonl_records::(&path, &file, stale_len, 5, "audit"); + fs::remove_dir_all(root).ok(); + assert_eq!(audit_messages(&recent?), ["a", "b"]); + Ok(()) + } + + #[test] + fn reverse_lines_refuses_to_splice_across_a_mid_scan_shrink() -> Result<()> { + let (root, _paths) = temp_app_paths("reverse-lines-shrink"); + fs::create_dir_all(&root)?; + let path = root.join("lines.jsonl"); + fs::write(&path, "aaaa\nbbbb\ncccc")?; + let file = fs::File::open(&path)?; + let mut lines = ReverseLines::new(&path, &file, 14, 4); + // As if the first step had read the last chunk, "cccc" (no `\n` yet), + // and the file then shrank below the next chunk's end. + lines.pending = b"cccc".to_vec(); + lines.pos = 10; + let shrunk = fs::OpenOptions::new() + .write(true) + .open(&path) + .and_then(|f| f.set_len(7)); + let next = lines.next_line(); + fs::remove_dir_all(root).ok(); + shrunk?; + assert!( + next.is_err(), + "a short read after the first must not be spliced" + ); + Ok(()) + } + + #[test] + fn readers_keep_an_unterminated_last_line_that_parses() -> Result<()> { + let (root, paths) = temp_app_paths("unterminated-complete"); + seed_audit_store(&paths, &format!("{}\n{}", audit_line("a"), audit_line("b")))?; + let audit = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert_eq!(audit_messages(&audit?), ["a", "b"]); + Ok(()) + } + + #[test] + fn append_cuts_a_torn_last_line_left_by_a_killed_writer() -> Result<()> { + let (root, paths) = temp_app_paths("append-torn-tail"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + append_audit_event(&paths, &audit_record("b"))?; + let text = fs::read_to_string(paths.audit_events_path()); + fs::remove_dir_all(root).ok(); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + Ok(()) + } + + #[test] + fn append_terminates_a_complete_unterminated_last_line() -> Result<()> { + let (root, paths) = temp_app_paths("append-unterminated"); + seed_audit_store(&paths, &audit_line("a"))?; + append_audit_event(&paths, &audit_record("b"))?; + let text = fs::read_to_string(paths.audit_events_path()); + fs::remove_dir_all(root).ok(); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + Ok(()) + } + + #[test] + fn append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly() -> Result<()> { + let (root, paths) = temp_app_paths("append-long-torn-tail"); + let long_torn = format!("{{\"message\":\"{}", "x".repeat(3 * 64 * 1024)); + seed_audit_store(&paths, &format!("{}\n{long_torn}", audit_line("a")))?; + append_audit_event(&paths, &audit_record("b"))?; + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert_eq!(audit_messages(&recent?), ["a", "b"]); + Ok(()) + } + + #[test] + fn the_store_lock_is_a_sibling_of_the_store() { + let paths = AppPaths { + config_dir: PathBuf::from("c"), + data_dir: PathBuf::from("d"), + cache_dir: PathBuf::from("k"), + }; + assert_eq!( + jsonl_lock_path(&paths.audit_events_path()), + paths.audit_dir().join("events.jsonl.lock") + ); + assert_eq!( + jsonl_lock_path(&paths.automation_proposals_path()), + paths.automations_dir().join("proposals.jsonl.lock") + ); + } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 7a5f5fd9af..f2963bc520 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -354,6 +354,10 @@ pub fn unix_time_millis() -> u128 { /// temp file and losing one of the two updates. /// /// Not guaranteed: +/// - Serializing writers: this takes no lock, so two read-modify-write +/// cycles that overlap still lose one update (last rename wins). A caller +/// whose new content depends on the old must hold a lock across the whole +/// cycle, as the JSONL proposal store does (`automation::jsonl_lock_path`). /// - Durability: the bytes are not fsynced before the rename, so a power /// loss can still surface the new name with unflushed contents. /// - Replacing a destination another process holds open without delete From cdde97c7e792154f847d2b221933862fa4326a63 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 08:03:58 +0000 Subject: [PATCH 2/8] ROCMAI-721: repair a torn JSONL tail by rename, not truncate, and fix the fallout A Copilot review on #738 found that cutting a torn tail in place and then appending as a separate write could let a lock-free reverse-scan reader observe the store shrink and regrow mid-scan, splicing bytes read at two different moments into something that parses as a record nobody wrote. Torn-tail repair now folds the cut into the append itself: the clean prefix plus the new record are written as one atomic rename over the store (`write_atomic`), so a reader only ever sees the pre-repair length or the post-append length. Removing the old rollback-on-failed-write (needed so a repair-in-place couldn't itself race a reader) opened a narrower gap: a write that failed one byte short of its own trailing `\n` left a complete, valid record on disk while still reporting `Err`, which would make a caller that retries on error write a duplicate. Appending now checks what actually landed before deciding, and only reports failure when the record genuinely didn't make it. Also fixes two pre-existing doc comments (process.rs, examine.feature) that still described appends as rewriting the whole store each time, and deduplicates the Windows file-identity test helper into test_support.rs. A torn-tail repair that can't complete its rename (sustained disk pressure, or a Windows handle without delete-sharing) still has no fallback and keeps retrying the same whole-store rewrite; tracked as a follow-up in ROCm/rocm-cli#748 rather than folded into this fix. Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 401 ++++++++++++++++---- crates/rocm-core/src/config.rs | 38 +- crates/rocm-core/src/process.rs | 13 +- crates/rocm-core/src/test_support.rs | 38 ++ tests/e2e-cucumber/features/examine.feature | 9 +- 5 files changed, 385 insertions(+), 114 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index d2b8d1a17f..3c094e0abf 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -12,14 +12,21 @@ //! //! The `rocm` CLI and `rocmd` write these stores concurrently, so every //! mutation of a store holds its sibling `.lock` file (see `jsonl_lock_path`). -//! Appending a record is one append-mode write of one line — its cost does not -//! grow with the store — rolled back if the write fails; a proposal status -//! update rewrites the store via `write_atomic`. Readers take no lock, and an -//! unterminated last line that is not valid JSON — an append caught mid-write — -//! is skipped rather than reported as corruption. The stores are not rotated -//! or capped. `AutomationTriggerEvent` is also defined here, but this -//! module persists none of it: it is the in-memory trigger type passed -//! between `rocmd`'s watcher and webhook plumbing (`apps/rocmd/src/watchers.rs`, +//! Appending a record is, in the common case, one append-mode write of one +//! line — its cost does not grow with the store. A torn tail — crash debris, +//! or left by a write that failed partway — is cut by the next append rather +//! than truncated in place: that append writes the store's clean prefix plus +//! its own new record as a single rewrite via `write_atomic`, so a lock-free +//! reader never observes a file shrunk mid-repair. That rewrite costs O(store) +//! rather than O(record), the same as a proposal status update, which rewrites +//! the whole store via `write_atomic` too. Nothing in this module ever +//! shrinks a store in place. Readers take no lock, and an unterminated last +//! line that is not valid JSON — an append caught mid-write — is skipped +//! rather than reported as corruption. The stores are not rotated or capped. +//! +//! `AutomationTriggerEvent` is also defined here, but this module persists +//! none of it: it is the in-memory trigger type passed between `rocmd`'s +//! watcher and webhook plumbing (`apps/rocmd/src/watchers.rs`, //! `apps/rocmd/src/webhook.rs`), never serialized. use anyhow::{Context, Result, bail}; @@ -219,10 +226,12 @@ pub struct AuditEventRecord { /// Lock file serializing every mutation of the JSONL store at `path` — a /// sibling named `.lock` (e.g. `events.jsonl.lock`). /// -/// Appends and the proposal-status rewrite both take it. The rewrite needs it -/// for its read-modify-write; appends need it too, because the rewrite -/// replaces the store by renaming a new file over it, and an append that had -/// already opened the old file would land in the replaced copy and vanish. +/// Appends and the proposal-status rewrite both take it. The proposal +/// rewrite needs it for its read-modify-write; an append needs it both to +/// serialize against other appends and because it can itself become a +/// rewrite — a torn tail is repaired by renaming a new file over the store +/// (see [`TailState::Torn`]) — and an append that had already opened the old +/// file would otherwise land in the replaced copy and vanish. fn jsonl_lock_path(path: &Path) -> PathBuf { let mut name = path.file_name().unwrap_or_default().to_os_string(); name.push(".lock"); @@ -234,15 +243,21 @@ const JSONL_SCAN_CHUNK: u64 = 64 * 1024; /// Append one JSONL `record` to `path`, creating the file if needed. /// -/// Holds the store's [`jsonl_lock_path`] lock and writes the line with a -/// single append-mode write, so the cost no longer grows with the store and -/// a concurrent writer can never drop this record (or have its own record -/// dropped by this one), across threads and processes alike. +/// Holds the store's [`jsonl_lock_path`] lock. The common case — the store +/// already ends on a line boundary — is a single append-mode write of one +/// line, so the cost no longer grows with the store and a concurrent writer +/// can never drop this record (or have its own record dropped by this one), +/// across threads and processes alike. /// -/// A failed write is rolled back to the pre-append length, so a full disk -/// never leaves a torn line behind. A torn line left by a writer that was -/// killed mid-write is cleaned up by the next append: see -/// [`repair_unterminated_tail`]. +/// A write that fails partway leaves whatever it already wrote as a new torn +/// tail — the same debris a killed writer leaves — rather than rolling it +/// back: seeing `Err` here means this specific record did not land (see +/// [`append_or_tolerate_a_missing_newline`] for the one exception, where it +/// did land despite the error), and the debris itself is discarded by the +/// *next* append rather than being cut first as a separate step — see +/// [`inspect_tail`] and [`TailState::Torn`]. A reader already skips the +/// debris in the meantime. Nothing in this function ever truncates the live +/// file, so a lock-free reader never observes it shrink. fn append_jsonl_record(path: &Path, record: &T, context_label: &str) -> Result<()> { let mut line = serde_json::to_vec(record) .with_context(|| format!("failed to serialize {context_label}"))?; @@ -255,35 +270,123 @@ fn append_jsonl_record(path: &Path, record: &T, context_label: &st .create(true) .open(path) .with_context(|| format!("failed to open {}", path.display()))?; - let (start, needs_separator) = repair_unterminated_tail(path, &file)?; - if needs_separator { - line.insert(0, b'\n'); - } - if let Err(error) = (&file).write_all(&line) { - // Best effort: if this fails too, the torn tail is still repaired by - // the next append, and readers already skip it. - let _ = truncate_file(path, start); - return Err(error).with_context(|| format!("failed to append to {}", path.display())); + match inspect_tail(path, &file)? { + TailState::Torn(tail_start) => { + // A torn tail is debris — crash or a failed write — that must be + // cut before this record lands. Cutting it in place and then + // appending as a second step would briefly shrink the file for + // any lock-free reader mid-scan, and regrowing it afterward can + // make that scan splice bytes read at two different moments into + // something that parses as a record nobody wrote (the hazard + // `ReverseLines::next_line`'s short-read guard exists for). + // Replacing the whole file atomically instead means a reader + // only ever sees the pre-repair length or this append's, never a + // length in between. + let mut content = read_range(path, &file, 0, tail_start)?; + content.extend_from_slice(&line); + crate::write_atomic(path, &content, "jsonl")?; + } + TailState::ValidUnterminated(start) => { + line.insert(0, b'\n'); + append_or_tolerate_a_missing_newline(path, &file, &line, start)?; + } + TailState::Clean(start) => { + append_or_tolerate_a_missing_newline(path, &file, &line, start)?; + } } Ok(()) } -/// Make the store at `path` end on a line boundary before an append, returning -/// the length to roll back to if the append fails and whether the new line -/// needs a leading `\n`. +/// Append-mode write of `line`, starting at offset `start`. /// -/// Must be called with the store's lock held, so no other writer is mid-write -/// and an unterminated last line is either crash debris (a writer killed -/// mid-write: not valid JSON, cut off) or a complete record whose newline was -/// lost to a hand edit (valid JSON, kept and terminated). -fn repair_unterminated_tail(path: &Path, file: &fs::File) -> Result<(u64, bool)> { +/// A write can fail after landing every byte of `line` except its final +/// `\n` (e.g. the disk fills up one byte short). That is not torn debris: +/// it is a complete, valid record, and readers already treat a complete +/// record missing only its trailing newline as fully stored (see +/// `readers_keep_an_unterminated_last_line_that_parses`). Reporting that as +/// a failure would tell a caller that retries on error to write the same +/// record again, leaving a duplicate the store has no way to detect or +/// remove — so this checks what actually landed before deciding. +fn append_or_tolerate_a_missing_newline( + path: &Path, + file: &fs::File, + line: &[u8], + start: u64, +) -> Result<()> { + let mut file = file; + let Err(error) = file.write_all(line) else { + return Ok(()); + }; + let landed = (|| -> Result> { + let end = file + .metadata() + .with_context(|| format!("failed to stat {}", path.display()))? + .len() + .max(start); + read_range(path, file, start, end) + })(); + match landed { + Ok(landed) if landed_despite_the_error(&landed, line) => Ok(()), + Ok(_) => Err(error).with_context(|| format!("failed to append to {}", path.display())), + // The write itself is still the more informative failure (e.g. the + // disk is full); checking whether it landed anyway is this + // function's own extra step, so losing that check is a detail + // folded into the same report rather than the headline error. + Err(check_error) => Err(error).with_context(|| { + format!( + "failed to append to {} ({check_error:#} while checking whether it landed anyway)", + path.display() + ) + }), + } +} + +/// Whether `landed` — the bytes a failed append-mode write actually put on +/// disk, read back from where it started — show `line` arrived intact +/// except possibly for its own trailing `\n`. Split out from +/// [`append_or_tolerate_a_missing_newline`] so the decision itself can be +/// tested without inducing a real write failure. +fn landed_despite_the_error(landed: &[u8], line: &[u8]) -> bool { + let without_trailing_newline = &line[..line.len().saturating_sub(1)]; + landed.len() >= without_trailing_newline.len() + && landed[..without_trailing_newline.len()] == *without_trailing_newline +} + +/// How [`inspect_tail`] found the store's current last line, ahead of an +/// append. The `Clean`/`ValidUnterminated` payload is the offset this +/// append's own bytes will start at, used by +/// [`append_or_tolerate_a_missing_newline`] to tell a complete record that +/// merely failed to land its own trailing newline from one that is actually +/// missing. +enum TailState { + /// Empty, or already ends on a line boundary. + Clean(u64), + /// A complete record, or a blank (whitespace-only) line, lost its + /// trailing `\n` — to a hand edit, or to a write that landed every byte + /// except that final one; the new line needs a leading `\n`. + ValidUnterminated(u64), + /// Debris — crash, or a write that failed partway through a genuinely + /// incomplete record — starting at this offset. Must be cut before the + /// next record lands, but never by truncating in place — see + /// [`append_jsonl_record`]. + Torn(u64), +} + +/// Classify the store's current last line ahead of an append. +/// +/// Must be called with the store's lock held, so no other writer is +/// mid-write and an unterminated last line is either debris (not valid JSON, +/// cut off — a writer killed mid-write, or one whose own append failed +/// partway) or a complete record or blank line whose newline was lost to a +/// hand edit or a write that failed one byte short (kept and terminated). +fn inspect_tail(path: &Path, file: &fs::File) -> Result { let len = file .metadata() .with_context(|| format!("failed to stat {}", path.display()))? .len(); // The common case — empty, or ending on a line boundary — costs one byte. if len == 0 || read_range(path, file, len - 1, len)? == b"\n" { - return Ok((len, false)); + return Ok(TailState::Clean(len)); } let mut tail = Vec::new(); let mut tail_start = len; @@ -299,21 +402,24 @@ fn repair_unterminated_tail(path: &Path, file: &fs::File) -> Result<(u64, bool)> tail_start = chunk_start; } if tail.is_empty() { - return Ok((len, false)); + return Ok(TailState::Clean(len)); } if is_blank(&tail) || !is_torn_record(&tail) { - return Ok((len, true)); + return Ok(TailState::ValidUnterminated(len)); } - truncate_file(path, tail_start)?; - Ok((tail_start, false)) + Ok(TailState::Torn(tail_start)) } /// Read bytes `[start, end)` of `file`. /// -/// Returns fewer bytes if the file now ends before `end`: a lock-free reader -/// can race a writer cutting a torn last line (see -/// [`repair_unterminated_tail`]), and the bytes cut are never a complete -/// record, so what is still there is all the caller needs. +/// Returns fewer bytes if the file now ends before `end`. Nothing this +/// module's own writers do can shrink a store in place — an append only ever +/// grows it, and a torn-tail repair replaces it wholesale via `write_atomic` +/// rather than truncating the live file (see [`inspect_tail`] and +/// [`TailState::Torn`]) — so a short read here means the store was edited or +/// truncated out-of-band, a case this module does not try to recover from +/// cleanly: the caller treats what is still there the same as any other +/// unexpectedly-ending tail. fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result> { let mut reader = file; reader @@ -327,17 +433,6 @@ fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result Result<()> { - fs::OpenOptions::new() - .write(true) - .open(path) - .and_then(|file| file.set_len(len)) - .with_context(|| format!("failed to truncate {}", path.display())) -} - /// Parse a whole JSONL store, pairing each record with its line index. /// /// Readers take no lock, so one may catch an append mid-write: a last line @@ -382,10 +477,12 @@ fn parse_jsonl_records( /// [`parse_jsonl_records`], an unterminated last line that is not valid JSON is /// an append caught mid-write and is skipped. /// -/// The scan takes no lock, so a writer may cut a torn last line (see -/// [`repair_unterminated_tail`]) and append again while it runs. A scan that -/// fails while the file's length moved under it is retried from the new end, -/// so that race never surfaces as an error. +/// The scan takes no lock. A concurrent append only ever grows the file (see +/// [`append_jsonl_record`]); nothing this module's own writers do shrinks it +/// in place. The retry below is for a store edited or truncated out-of-band +/// — by a person, not a concurrent writer — while a scan is in flight: a +/// scan that fails while the file's length moved under it is retried from +/// the new end, so that still never surfaces as an error. fn load_recent_jsonl_records( path: &Path, limit: usize, @@ -507,9 +604,11 @@ impl<'a> ReverseLines<'a> { } let start = self.pos.saturating_sub(self.chunk); let mut bytes = read_range(self.path, self.file, start, self.pos)?; - // A short first read just means a torn tail was cut (see - // `read_range`). Short after that, the bytes already read no longer - // continue from these, so splicing them would invent a line. + // A short first read just means the store was edited or + // truncated out-of-band (see `read_range`); nothing a concurrent + // writer does can shrink it in place. Short after that, the + // bytes already read no longer continue from these, so splicing + // them would invent a line. if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { bail!("{} shrank while it was being read", self.path.display()); } @@ -1294,8 +1393,9 @@ mod tests { Ok(()) } - // A lock-free reader that read the length just before a writer cut a torn - // tail sees the file end early; that is not an error. + // A lock-free reader that measured a stale, larger length — because the + // store was edited out-of-band, not because a concurrent writer shrank + // it — sees the file end early; that is not an error. #[test] fn a_scan_that_finds_the_file_shorter_than_it_was_still_succeeds() -> Result<()> { let (root, paths) = temp_app_paths("scan-shrunk"); @@ -1384,6 +1484,175 @@ mod tests { Ok(()) } + #[cfg(unix)] + #[test] + fn append_of_a_torn_tail_replaces_rather_than_truncates_the_file_in_place() -> Result<()> { + use std::os::unix::fs::MetadataExt; + + // The inode is the observable difference between the two ways to + // repair a torn tail: truncating it in place keeps rewriting the + // SAME file, so a lock-free reader mid-scan can see it briefly + // shorter than before the repair and then longer again once the new + // record lands — the shrink-and-regrow window a reverse scan can + // misread as a record nobody wrote (see `append_jsonl_record`). A + // rename publishes a DIFFERENT file over the name, so a reader only + // ever sees the pre-repair length or the post-append length, never + // one in between. + // + // `append_cuts_a_torn_last_line_left_by_a_killed_writer` above only + // checks the final bytes, which holds just as well for an in-place + // truncate-then-write, so it cannot tell the two apart. + // + // The `#[cfg(windows)]` function of the same name below asserts the + // same guarantee through `GetFileInformationByHandle`. + let (root, paths) = temp_app_paths("append-torn-tail-replaces"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = fs::metadata(&path)?.ino(); + + append_audit_event(&paths, &audit_record("b"))?; + + let after = fs::metadata(&path)?.ino(); + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_ne!( + before, after, + "repairing a torn tail kept the same inode, so it truncated the \ + live file in place instead of replacing it" + ); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + Ok(()) + } + + #[cfg(windows)] + #[test] + fn append_of_a_torn_tail_replaces_rather_than_truncates_the_file_in_place() -> Result<()> { + use crate::test_support::windows_file_identity; + + // The Windows counterpart of the Unix test above. The file index + // together with the volume serial number is this platform's file + // identity, the way an inode is on Unix: an in-place truncate keeps + // that pair, while a repair-by-rename gives the live path a + // different one. + let (root, paths) = temp_app_paths("append-torn-tail-replaces"); + let torn = &audit_line("torn")[..20]; + seed_audit_store(&paths, &format!("{}\n{torn}", audit_line("a")))?; + let path = paths.audit_events_path(); + let before = windows_file_identity(&path)?; + + append_audit_event(&paths, &audit_record("b"))?; + + let after = windows_file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_ne!( + before, after, + "repairing a torn tail kept the same file identity, so it \ + truncated the live file in place instead of replacing it" + ); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + Ok(()) + } + + #[cfg(unix)] + #[test] + fn append_to_a_clean_or_unterminated_store_keeps_the_same_file() -> Result<()> { + use std::os::unix::fs::MetadataExt; + + // The counterpart of the torn-tail-repair test above, for the other + // two `TailState` arms: with no torn tail to repair — the store either + // already ends on a line boundary, or ends in a complete record + // missing just its `\n` — an append must stay a plain append-mode + // write on the SAME file, not the atomic replace the torn-tail case + // takes. An inode change here would mean every append costs a + // full-file rewrite, which is exactly the per-append cost this + // store's design exists to avoid. + for seed in [ + format!("{}\n", audit_line("a")), // Clean: ends on a line boundary. + audit_line("a"), // ValidUnterminated: no trailing `\n`. + ] { + let (root, paths) = temp_app_paths("append-clean-same-file"); + seed_audit_store(&paths, &seed)?; + let path = paths.audit_events_path(); + let before = fs::metadata(&path)?.ino(); + + append_audit_event(&paths, &audit_record("b"))?; + + let after = fs::metadata(&path)?.ino(); + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!( + before, after, + "an append with no torn tail to repair replaced the file \ + instead of appending to it in place, seed {seed:?}" + ); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + } + Ok(()) + } + + #[cfg(windows)] + #[test] + fn append_to_a_clean_or_unterminated_store_keeps_the_same_file() -> Result<()> { + use crate::test_support::windows_file_identity; + + // The Windows counterpart of the Unix test above. + for seed in [ + format!("{}\n", audit_line("a")), // Clean: ends on a line boundary. + audit_line("a"), // ValidUnterminated: no trailing `\n`. + ] { + let (root, paths) = temp_app_paths("append-clean-same-file"); + seed_audit_store(&paths, &seed)?; + let path = paths.audit_events_path(); + let before = windows_file_identity(&path)?; + + append_audit_event(&paths, &audit_record("b"))?; + + let after = windows_file_identity(&path)?; + let text = fs::read_to_string(&path); + fs::remove_dir_all(root).ok(); + assert_eq!( + before, after, + "an append with no torn tail to repair replaced the file \ + instead of appending to it in place, seed {seed:?}" + ); + assert_eq!(text?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + } + Ok(()) + } + + #[test] + fn landed_despite_the_error_tolerates_only_a_missing_trailing_newline() { + let line = b"{\"a\":1}\n"; + assert!( + landed_despite_the_error(&line[..line.len() - 1], line), + "a write that failed one byte short of the trailing newline must \ + still count as landed: the record is complete and valid, and a \ + caller that reports this as a failure and retries would write \ + it twice" + ); + assert!( + landed_despite_the_error(line, line), + "the full line landing must also count, even if write_all \ + somehow still reported an error for it" + ); + assert!( + !landed_despite_the_error(&line[..line.len() - 2], line), + "missing more than just the trailing newline is genuine torn \ + debris, not a complete record" + ); + assert!( + !landed_despite_the_error(b"", line), + "nothing landing at all is not a complete record" + ); + assert!( + !landed_despite_the_error(b"{\"a\":2}\n", line), + "different bytes landing at the same length must not be \ + mistaken for this record having landed" + ); + } + #[test] fn the_store_lock_is_a_sibling_of_the_store() { let paths = AppPaths { diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index 860ddcf0e7..b1efe4b1ab 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -1143,45 +1143,11 @@ last_installed_runtime_id = "therock-release" Ok(()) } - /// The volume serial number paired with the 64-bit file index: this - /// platform's file identity, the way an inode is on Unix. - /// - /// Read through `GetFileInformationByHandle` rather than - /// `std::os::windows::fs::MetadataExt`, whose `volume_serial_number` and - /// `file_index` accessors are unstable behind the `windows_by_handle` - /// feature gate and so cannot be called on the pinned stable toolchain. - #[cfg(windows)] - #[allow(unsafe_code)] // Win32 FFI - fn windows_file_identity(path: &Path) -> Result<(u32, u64)> { - use anyhow::bail; - use std::os::windows::io::AsRawHandle; - use windows_sys::Win32::Foundation::HANDLE; - use windows_sys::Win32::Storage::FileSystem::{ - BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, - }; - - let file = - fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; - // All-POD `repr(C)` record, fully overwritten by the call below on - // success and never read on failure. - let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; - let handle = file.as_raw_handle() as HANDLE; - if unsafe { GetFileInformationByHandle(handle, &raw mut info) } == 0 { - bail!( - "failed to read the file identity of {}: {}", - path.display(), - std::io::Error::last_os_error() - ); - } - Ok(( - info.dwVolumeSerialNumber, - (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow), - )) - } - #[cfg(windows)] #[test] fn saving_the_config_replaces_the_file_rather_than_rewriting_it_in_place() -> Result<()> { + use crate::test_support::windows_file_identity; + // The Windows counterpart of the Unix test above. The file index // together with the volume serial number is this platform's file // identity, the way an inode is on Unix: a `fs::write` over the live diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 0c715a5fd3..724807ff96 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -637,10 +637,9 @@ const DRAIN_GRACE: Duration = Duration::from_millis(200); /// The bound matters because the message does not stop at a terminal. `rocmd` /// renders the whole error chain into the bridge snapshot's `note`, and the /// GPU-metrics watcher copies that note into the automation-event and audit -/// JSONL files once a minute — files whose append helpers rewrite the whole -/// file each time. Unbounded, a child that prints a megabyte and then stalls -/// past its timeout grows both logs by that much per minute, and every -/// subsequent append costs more than the last. +/// JSONL files once a minute — stores that are never rotated or capped. +/// Unbounded, a child that prints a megabyte and then stalls past its +/// timeout grows both logs by that much per minute, forever. /// /// Sized to still hold a real stderr backtrace: the point is to stop a /// payload-sized quote, not to shorten diagnostics. @@ -1866,9 +1865,9 @@ mod tests { // // Unbounded, this message travels: `rocmd` renders the error chain // into the bridge snapshot's `note`, and the GPU-metrics watcher - // writes that note into two JSONL logs every 60s, rewriting each - // file whole. A real amd-smi on a multi-GPU host prints well past - // this fixture's size. + // writes that note into two JSONL logs every 60s, neither ever + // rotated or capped. A real amd-smi on a multi-GPU host prints + // well past this fixture's size. let mut command = Command::new(shell); command.args([ "-c", diff --git a/crates/rocm-core/src/test_support.rs b/crates/rocm-core/src/test_support.rs index 36d047310a..2671ccae1d 100644 --- a/crates/rocm-core/src/test_support.rs +++ b/crates/rocm-core/src/test_support.rs @@ -13,6 +13,44 @@ use crate::{AppPaths, ManagedServiceRecord, unix_time_millis}; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; +/// The volume serial number paired with the 64-bit file index: this +/// platform's file identity, the way an inode is on Unix. Shared by every +/// test that tells a publish-by-rename apart from a rewrite in place. +/// +/// Read through `GetFileInformationByHandle` rather than +/// `std::os::windows::fs::MetadataExt`, whose `volume_serial_number` and +/// `file_index` accessors are unstable behind the `windows_by_handle` +/// feature gate and so cannot be called on the pinned stable toolchain. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +pub(crate) fn windows_file_identity(path: &std::path::Path) -> anyhow::Result<(u32, u64)> { + use anyhow::{Context, bail}; + use std::fs; + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Foundation::HANDLE; + use windows_sys::Win32::Storage::FileSystem::{ + BY_HANDLE_FILE_INFORMATION, GetFileInformationByHandle, + }; + + let file = + fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; + // All-POD `repr(C)` record, fully overwritten by the call below on + // success and never read on failure. + let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() }; + let handle = file.as_raw_handle() as HANDLE; + if unsafe { GetFileInformationByHandle(handle, &raw mut info) } == 0 { + bail!( + "failed to read the file identity of {}: {}", + path.display(), + std::io::Error::last_os_error() + ); + } + Ok(( + info.dwVolumeSerialNumber, + (u64::from(info.nFileIndexHigh) << 32) | u64::from(info.nFileIndexLow), + )) +} + pub(crate) fn temp_app_paths(name: &str) -> (PathBuf, AppPaths) { let root = workspace_test_artifact_dir().join(format!("rocm-core-{name}-{}", unique_suffix())); let paths = AppPaths { diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 38a8390118..59d02234f0 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -367,11 +367,10 @@ Feature: GPU detection and system inspection # With an amd-smi that prints a snapshot's worth of JSON and only then # stalls, that quote is the entire payload, and the note is not a terminal # message: `record_gpu_metrics` copies it into the automation-event and audit - # JSONL files every 60s, and both append helpers rewrite the whole file each - # time. Unbounded, the logs grew by the payload's size every minute and each - # write cost more than the last. So the note must name the timeout, keep - # enough of the child's output to be worth reading, say that it was shortened - # -- and stay small. + # JSONL files every 60s, neither ever rotated or capped. Unbounded, the logs + # grew by the payload's size every minute, forever. So the note must name + # the timeout, keep enough of the child's output to be worth reading, say + # that it was shortened -- and stay small. # # Asserted here rather than only as a unit test because the note's journey is # the point: the bound lives in `rocm-core`'s `run_with_timeout` and the From 64e392d30fa799bd5d6e41c16becf27183cf2357 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 11:17:02 +0000 Subject: [PATCH 3/8] ROCMAI-721: document FileLock's JSONL-store use in process.rs An automated review on #738 noted that FileLock's doc comment listed the daemon autostart and managed-launch GPU claim as its use sites but not the per-store JSONL lock this PR added. Reworded both the struct doc and the module header to name it alongside the others, and hedged the list with "e.g." rather than letting it read as an exhaustive call-site catalog (it already wasn't one: it never listed the `uv` install lock either, and main.rs's own lock-reachability note is the authoritative enumeration). Signed-off-by: Jussi Elo --- crates/rocm-core/src/process.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 724807ff96..afbea28608 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -6,8 +6,9 @@ //! //! Hidden-console/detached spawn and exit-wait on Windows, process/process-tree //! termination on both platforms, and the exclusive `FileLock` used to -//! serialize the daemon autostart path, the managed-launch GPU claim, and -//! `uv` installs across processes. +//! serialize check-then-act sequences over shared on-disk state across +//! processes — e.g. the daemon autostart path, the managed-launch GPU claim, +//! a `uv` install, and each JSONL store's append/rewrite. //! //! [`run_with_timeout`] is the bulk of the file: run a command to completion //! with both streams captured and a real time bound. A thread per pipe drains @@ -387,9 +388,11 @@ pub fn process_is_running(pid: u32) -> bool { /// Wraps the standard-library file lock (`std::fs::File::lock`), so the exclusion /// holds between *separate `rocm` processes*, not just threads: each caller opens /// the same lock-file path and only one can hold the lock at a time. It exists to -/// serialize check-then-act sequences over shared on-disk state — the daemon -/// autostart decision and the managed-serve GPU select-then-claim — so two -/// concurrent invocations cannot both pass the same TOCTOU check. +/// serialize check-then-act sequences over shared on-disk state — e.g. the +/// daemon autostart decision, the managed-serve GPU select-then-claim, a +/// `uv` install, and each JSONL store's append-or-repair and read-modify-write +/// (see `jsonl_lock_path` in `automation.rs`) — so two concurrent invocations +/// cannot both pass the same check, or interleave a write with another's. /// /// The lock is released when the guard is dropped, and by the OS if the process /// exits while holding it (so a crashed holder never wedges the next caller). From e8beefa297cc4cacf80d99a0da08f6305cd4e86b Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 11:17:17 +0000 Subject: [PATCH 4/8] ROCMAI-721: have inspect_tail reuse ReverseLines's backward scan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An automated review on #738 noted that inspect_tail's own backward chunk-scan loop duplicated ReverseLines's "find the last line scanning backwards in fixed-size chunks" logic as a second, independently maintained implementation. The segment inspect_tail wants (whatever follows the last `\n`) is exactly ReverseLines's first yielded line, so it now calls that instead. Reusing it meant giving next_line() a way to report where the line it returns actually starts: it now yields `(start, bytes)`, with `start` derived from the struct's own tracked read position rather than from a length the caller measured before the scan began (which can go stale if the store shrinks out-of-band mid-scan — the first draft of this reuse got that wrong, caught by a pre-commit review round-trip, before it could affect inspect_tail's torn-tail repair offset). Strengthened reverse_lines_matches_a_forward_split_at_every_chunk_size to also check every yielded line's reported start, and added reverse_lines_tolerates_a_shrink_before_the_first_read to pin that a shrink before any chunk has been read (not guarded, since there is no prior `pending` to misalign against) still reports the real content and offset rather than the stale one. Also corrected two comments nearby (the shrink guard's own, and read_range's) that described the guard's condition and its callers' behavior less precisely than the code. Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 98 ++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 26 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 3c094e0abf..01004667aa 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -388,19 +388,11 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { if len == 0 || read_range(path, file, len - 1, len)? == b"\n" { return Ok(TailState::Clean(len)); } - let mut tail = Vec::new(); - let mut tail_start = len; - while tail_start > 0 { - let chunk_start = tail_start.saturating_sub(JSONL_SCAN_CHUNK); - let chunk = read_range(path, file, chunk_start, tail_start)?; - if let Some(newline) = chunk.iter().rposition(|&byte| byte == b'\n') { - tail.splice(0..0, chunk[newline + 1..].iter().copied()); - tail_start = chunk_start + newline as u64 + 1; - break; - } - tail.splice(0..0, chunk); - tail_start = chunk_start; - } + // The segment after the last `\n` is exactly `ReverseLines`'s first line, + // so reuse its backward scan rather than re-implementing it here. + let (tail_start, tail) = ReverseLines::new(path, file, len, JSONL_SCAN_CHUNK) + .next_line()? + .unwrap_or_default(); if tail.is_empty() { return Ok(TailState::Clean(len)); } @@ -418,8 +410,10 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// rather than truncating the live file (see [`inspect_tail`] and /// [`TailState::Torn`]) — so a short read here means the store was edited or /// truncated out-of-band, a case this module does not try to recover from -/// cleanly: the caller treats what is still there the same as any other -/// unexpectedly-ending tail. +/// cleanly. Most callers treat what is still there the same as any other +/// unexpectedly-ending tail; [`ReverseLines`] is the one exception, and +/// reports an error instead once it would have to splice such a read onto +/// bytes it already read, rather than risk inventing a line. fn read_range(path: &Path, file: &fs::File, start: u64, end: u64) -> Result> { let mut reader = file; reader @@ -526,14 +520,14 @@ fn scan_recent_jsonl_records( let mut records = Vec::new(); // The first segment is whatever follows the last `\n`: empty when the // file ends on a line boundary, else an unterminated last line. - if let Some(tail) = lines.next_line()? + if let Some((_, tail)) = lines.next_line()? && !is_blank(&tail) && !is_torn_record(&tail) { records.push(parse(&tail)?); } while records.len() < limit { - let Some(line) = lines.next_line()? else { + let Some((_, line)) = lines.next_line()? else { break; }; if !is_blank(&line) { @@ -586,7 +580,11 @@ impl<'a> ReverseLines<'a> { } } - fn next_line(&mut self) -> Result>> { + /// Yields `(start, bytes)`: `start` is the returned line's absolute file + /// offset, derived from `self.pos` (which tracks real, current file + /// offsets as actually read) rather than from a length measured earlier + /// by the caller — which may be stale by the time a scan reaches it. + fn next_line(&mut self) -> Result)>> { if self.done { return Ok(None); } @@ -596,19 +594,22 @@ impl<'a> ReverseLines<'a> { if let Some(newline) = self.pending.iter().rposition(|&byte| byte == b'\n') { let line = self.pending.split_off(newline + 1); self.pending.truncate(newline); - return Ok(Some(line)); + let start = self.pos + newline as u64 + 1; + return Ok(Some((start, line))); } if self.pos == 0 { self.done = true; - return Ok(Some(std::mem::take(&mut self.pending))); + return Ok(Some((0, std::mem::take(&mut self.pending)))); } let start = self.pos.saturating_sub(self.chunk); let mut bytes = read_range(self.path, self.file, start, self.pos)?; - // A short first read just means the store was edited or - // truncated out-of-band (see `read_range`); nothing a concurrent - // writer does can shrink it in place. Short after that, the - // bytes already read no longer continue from these, so splicing - // them would invent a line. + // A short read while `pending` is empty — nothing accumulated + // yet, whether from an earlier call or from this one — just + // means the store was edited or truncated out-of-band (see + // `read_range`); nothing a concurrent writer does can shrink it + // in place. Once `pending` holds any bytes, a short read means + // they no longer continue from it, so splicing them would + // invent a line. if bytes.len() as u64 != self.pos - start && !self.pending.is_empty() { bail!("{} shrank while it was being read", self.path.display()); } @@ -1278,10 +1279,27 @@ mod tests { for chunk in 1..=8 { let mut lines = ReverseLines::new(&path, &file, file.metadata()?.len(), chunk); let mut got = Vec::new(); - while let Some(line) = lines.next_line()? { + let mut starts = Vec::new(); + while let Some((start, line)) = lines.next_line()? { + starts.push(start); got.push(line); } assert_eq!(got, expected, "contents {contents:?}, chunk {chunk}"); + // Each line's reported start plus its own length must land + // exactly one byte before the previous (later-in-file) line's + // start — the byte in between is the `\n` that separated + // them — except the first line yielded here (the file's + // *last* line, with nothing after it to separate from), + // which must instead land exactly on the file's own length. + let mut expected_next = contents.len() as u64; + for (&start, line) in starts.iter().zip(&got) { + assert_eq!( + start + line.len() as u64, + expected_next, + "contents {contents:?}, chunk {chunk}: wrong start for {line:?}" + ); + expected_next = start.saturating_sub(1); + } } } fs::remove_dir_all(root).ok(); @@ -1439,6 +1457,34 @@ mod tests { Ok(()) } + // Unlike the shrink above, a shrink before any chunk has been read yet + // isn't guarded (there is no prior `pending` to misalign against), so it + // must not error — but the line it finds, and the offset it reports, + // still have to match where the content actually is now, not where the + // `len` passed to `new` (now stale) said it would be. + #[test] + fn reverse_lines_tolerates_a_shrink_before_the_first_read() -> Result<()> { + let (root, _paths) = temp_app_paths("reverse-lines-shrink-first-read"); + fs::create_dir_all(&root)?; + let path = root.join("lines.jsonl"); + fs::write(&path, "aaaa\nbbbb\ncccc")?; + let file = fs::File::open(&path)?; + // As if this had measured the file at its original 14 bytes, then the + // file shrank to 9 ("aaaa\nbbbb", no trailing "\ncccc") before this + // scan's first read. + fs::OpenOptions::new().write(true).open(&path)?.set_len(9)?; + let mut lines = ReverseLines::new(&path, &file, 14, 4); + let next = lines.next_line(); + fs::remove_dir_all(root).ok(); + let (start, line) = next?.expect("one line"); + assert_eq!(line, b"bbbb", "must read what is really there now"); + assert_eq!( + start, 5, + "offset must match the real content, not the stale length" + ); + Ok(()) + } + #[test] fn readers_keep_an_unterminated_last_line_that_parses() -> Result<()> { let (root, paths) = temp_app_paths("unterminated-complete"); From 3a07696ae1cb7e5ff9791c4a451c304a9529ea0f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 15:46:02 +0000 Subject: [PATCH 5/8] ROCMAI-721: cover the retry, partial-write, and torn-tail-race gaps review found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two human reviewers and this repo's automated pr-review-watcher independently flagged the same two blocking test-coverage gaps on #738, plus several smaller ones: - load_recent_jsonl_records's retry-after-a-failed-scan path was never exercised: every existing shrink test called scan_recent_jsonl_records or ReverseLines directly, bypassing the wrapper's retry entirely. Driving it needs the shrink to land strictly between two of ReverseLines's own chunked reads, which in turn needs a scan spanning more than one physical chunk — hence the new test_hooks seam (mirroring uv.rs's own test-hook pattern) and the pathologically long seeded record. - append_or_tolerate_a_missing_newline's glue around a real failed write was untested — only the pure landed_despite_the_error predicate had a direct test. Two new Unix tests force a genuine short write via RLIMIT_FSIZE (SIGXFSZ ignored): one where the write lands everything but its trailing newline (must still report Ok, with no duplicate), and one with zero slack, where the very first write(2) call lands nothing at all (must report a real failure, not fold it into a false success — closing a gap the first test alone couldn't catch). - No test raced a torn-tail repair against concurrent appenders, the exact scenario the per-store lock's own doc comment gives as its reason to exist; both existing concurrency tests started from an empty store. - A whitespace-only unterminated tail has no test pinning that it's kept and terminated rather than cut as torn debris (TailState::ValidUnterminated documents this, but nothing exercised it). - FileLock's doc comment didn't mention the per-store JSONL lock. - Two "redundant" guards (records.truncate(limit), the unterminated && check in parse_jsonl_records) are each provably dead in the current call graph, but are real safety nets against a future direct caller bypassing the guard that makes them dead today — kept, with a comment saying so instead of being removed. - drop(file) added before the Torn arm's write_atomic call in append_jsonl_record: the rename already succeeds with the old handle open (std's default Windows share mode includes delete-sharing), but this states that intent explicitly instead of leaning on the default. Every new test and comment went through several rounds of independent fresh review before landing; early drafts had real bugs (a stale-length offset computation, a chunk-size change that didn't achieve what it claimed, an overclaimed "only reader" comment, a signal/rlimit restore gap) caught and fixed before being committed. Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 435 +++++++++++++++++++++++++++++ 1 file changed, 435 insertions(+) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 01004667aa..be6a3f22cb 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -284,6 +284,16 @@ fn append_jsonl_record(path: &Path, record: &T, context_label: &st // length in between. let mut content = read_range(path, &file, 0, tail_start)?; content.extend_from_slice(&line); + // Done with the pre-repair file's handle before replacing it, + // rather than leaving that to whatever happens to still have + // `file` open at function return. The rename already succeeds + // with this handle open regardless — std opens files with + // `FILE_SHARE_DELETE` by default on Windows, so a rename isn't + // blocked by it (see `write_atomic`'s doc for the case that + // *isn't* covered: another process holding the destination open + // without delete-sharing) — but this states the intent rather + // than leaning on that default. + drop(file); crate::write_atomic(path, &content, "jsonl")?; } TailState::ValidUnterminated(start) => { @@ -442,6 +452,12 @@ fn parse_jsonl_records( path: &Path, context_label: &str, ) -> Result> { + // Redundant with "this is the last split segment and it's non-blank" + // below (splitting on `\n` makes the final segment empty exactly when + // `bytes` ends in one, so a non-blank final segment can only occur when + // it doesn't) — kept spelled out because it states directly what that + // arm is for, rather than leaning on the split/is_blank interaction to + // prove it. let unterminated = !bytes.is_empty() && !bytes.ends_with(b"\n"); let line_count = bytes.split(|&byte| byte == b'\n').count(); let mut records = Vec::new(); @@ -534,6 +550,12 @@ fn scan_recent_jsonl_records( records.push(parse(&line)?); } } + // No-op today: the loop above never pushes past `limit`, and the one + // production caller (`load_recent_jsonl_records`) already refuses + // `limit == 0` before calling in. Kept as a guard against a future + // direct caller that skips that guard — the tail-line push above the + // loop isn't itself gated by `limit`, so it would return one record for + // `limit == 0` without this. records.truncate(limit); records.reverse(); Ok(records) @@ -556,6 +578,14 @@ fn is_torn_record(line: &[u8]) -> bool { /// Yields the `\n`-separated segments of a file from last to first, reading /// it backwards in fixed-size chunks. The first segment yielded is the one /// after the final `\n` (empty when the file ends with one). +/// +/// How many chunks get read depends on the file's `\n` layout, not just its +/// length: a single line longer than `chunk` is rescanned from its own start +/// on every subsequent chunk read until its leading `\n` is found (each read +/// re-prepends onto the same growing `pending`), so one pathologically long +/// line costs O(length² / chunk) rather than O(length). Fine for this +/// module's records and the deliberately oversized test cases below; a +/// store whose lines could grow unbounded would need a different scan. struct ReverseLines<'a> { path: &'a Path, file: &'a fs::File, @@ -602,6 +632,8 @@ impl<'a> ReverseLines<'a> { return Ok(Some((0, std::mem::take(&mut self.pending)))); } let start = self.pos.saturating_sub(self.chunk); + #[cfg(test)] + test_hooks::before_scan_read(self.path); let mut bytes = read_range(self.path, self.file, start, self.pos)?; // A short read while `pending` is empty — nothing accumulated // yet, whether from an earlier call or from this one — just @@ -620,6 +652,54 @@ impl<'a> ReverseLines<'a> { } } +/// A seam for forcing an out-of-band shrink between two of `ReverseLines`'s +/// own chunked reads, so a test can drive `load_recent_jsonl_records`'s +/// retry-after-a-failed-scan path deterministically. Mirrors the +/// path-keyed-callback pattern in `uv.rs`'s `test_hooks`, but each hook is +/// one-shot and keyed by how many reads to let pass first: unlike an install +/// download's single commit point, a scan's vulnerable read (the one where +/// `pending` already holds bytes from an earlier read) is not the first one. +#[cfg(test)] +mod test_hooks { + use std::collections::HashMap; + use std::path::{Path, PathBuf}; + use std::sync::{Arc, LazyLock, Mutex, PoisonError}; + + type Hook = Arc; + + /// Keyed by path, value is (reads left to skip, hook to run at zero). + static BEFORE_SCAN_READ: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + + /// Run `hook` once, immediately before the `skip`-th subsequent physical + /// chunk read `ReverseLines` performs against `path` (`0` = the very next + /// read). Removed after it fires, so later reads of the same scan (or a + /// retried one) proceed unhooked. + pub(super) fn set_before_scan_read(path: &Path, skip: usize, hook: Hook) { + BEFORE_SCAN_READ + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(path.to_path_buf(), (skip, hook)); + } + + pub(super) fn before_scan_read(path: &Path) { + let mut hooks = BEFORE_SCAN_READ + .lock() + .unwrap_or_else(PoisonError::into_inner); + let Some((skip, hook)) = hooks.get_mut(path) else { + return; + }; + if *skip > 0 { + *skip -= 1; + return; + } + let hook = Arc::clone(hook); + hooks.remove(path); + drop(hooks); + hook(); + } +} + pub fn append_automation_event(paths: &AppPaths, event: &AutomationEventRecord) -> Result<()> { paths.ensure()?; append_jsonl_record( @@ -1092,6 +1172,78 @@ mod tests { Ok(()) } + /// Seed each of the three stores with one clean record followed by the + /// same shape of torn tail a killed writer leaves: a cut-off JSON + /// fragment with no trailing newline. + fn seed_torn_tail_in_every_store(paths: &AppPaths) -> Result<()> { + paths.ensure()?; + let torn_audit = &audit_line("torn")[..20]; + fs::write( + paths.audit_events_path(), + format!("{}\n{torn_audit}", audit_line("seed")), + )?; + let torn_event = &serde_json::to_string(&event_record("torn"))?[..20]; + fs::write( + paths.automation_events_path(), + format!( + "{}\n{torn_event}", + serde_json::to_string(&event_record("seed"))? + ), + )?; + let torn_proposal = &serde_json::to_string(&proposal_record("torn"))?[..20]; + fs::write( + paths.automation_proposals_path(), + format!( + "{}\n{torn_proposal}", + serde_json::to_string(&proposal_record("seed"))? + ), + )?; + Ok(()) + } + + // Regression test for the exact hazard jsonl_lock_path's own doc comment + // gives as its reason to exist: a torn-tail repair (the atomic-rename + // replace) racing a concurrent appender that already opened the + // pre-repair file. Neither existing concurrency test starts from a torn + // tail, so the Torn arm's rewrite path was never exercised concurrently + // until now. + #[test] + fn concurrent_appends_after_a_torn_tail_lose_no_records() -> Result<()> { + const WRITERS: usize = 8; + const APPENDS_PER_WRITER: usize = 25; + let (root, paths) = temp_app_paths("jsonl-torn-tail-concurrent"); + seed_torn_tail_in_every_store(&paths)?; + let barrier = std::sync::Barrier::new(WRITERS); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..WRITERS) + .map(|writer| { + let (paths, barrier) = (&paths, &barrier); + scope.spawn(move || -> Result<()> { + barrier.wait(); + for n in 0..APPENDS_PER_WRITER { + append_to_every_store(paths, &format!("t{writer}-{n}"))?; + } + Ok(()) + }) + }) + .collect(); + for handle in handles { + handle.join().expect("writer thread panicked")?; + } + Ok::<_, anyhow::Error>(()) + })?; + + let mut expected: Vec = vec!["seed".to_owned()]; + expected.extend( + (0..WRITERS) + .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("t{writer}-{n}"))), + ); + let stores = tags_in_every_store(&paths); + fs::remove_dir_all(root).ok(); + assert_every_store_holds_exactly(stores?, &expected); + Ok(()) + } + /// Env var that turns a re-exec of this test binary into one writer of /// [`concurrent_appends_from_many_processes_lose_no_records`]: its value is /// `|`. @@ -1154,6 +1306,230 @@ mod tests { Ok(()) } + /// Run `body` with `RLIMIT_FSIZE`'s soft limit temporarily lowered to + /// `limit_bytes`, restoring the original soft limit — leaving the hard + /// limit untouched throughout — before returning, so a coverage build's + /// own `.profraw` write at process exit (after this returns, outside + /// `body`'s control) isn't cut short by the same rlimit. `SIGXFSZ` is + /// ignored for the duration, so a write that would exceed the limit + /// returns `Err(EFBIG)` instead of killing the process. + #[cfg(unix)] + #[allow(unsafe_code)] // libc FFI: lower/restore RLIMIT_FSIZE and ignore SIGXFSZ for the duration + fn with_lowered_fsize_limit(limit_bytes: u64, body: impl FnOnce() -> Result<()>) -> Result<()> { + let mut original = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + unsafe { + if libc::getrlimit(libc::RLIMIT_FSIZE, &raw mut original) != 0 { + bail!("getrlimit failed: {}", std::io::Error::last_os_error()); + } + } + let lowered = libc::rlimit { + rlim_cur: limit_bytes as libc::rlim_t, + rlim_max: original.rlim_max, + }; + // The signal handler and the rlimit are restored unconditionally + // below, regardless of whether lowering the limit here succeeded or + // `body` even ran — this process stays around for the rest of the + // (child) test process's life, so a skipped restore on an early + // `bail!` would leave it running with SIGXFSZ permanently ignored. + let previous_handler = unsafe { libc::signal(libc::SIGXFSZ, libc::SIG_IGN) }; + let lowered_ok = unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &raw const lowered) == 0 }; + let result = if lowered_ok { + body() + } else { + Err(anyhow::anyhow!( + "setrlimit failed: {}", + std::io::Error::last_os_error() + )) + }; + let restored_ok = unsafe { libc::setrlimit(libc::RLIMIT_FSIZE, &raw const original) == 0 }; + unsafe { + libc::signal(libc::SIGXFSZ, previous_handler); + } + if !restored_ok { + bail!( + "restoring rlimit failed: {}", + std::io::Error::last_os_error() + ); + } + result + } + + // Regression test for append_or_tolerate_a_missing_newline's glue around + // a real failed write — not just the pure landed_despite_the_error + // predicate it calls, tested below in isolation with hand-built byte + // slices. An RLIMIT_FSIZE one byte under the record's full length + // reproduces, deterministically, the exact case that glue exists for. + // `write_all` loops over `write(2)`: its first call lands every byte up + // to the limit as a successful short write; its next call, with the one + // remaining byte (the trailing `\n`) now past the limit, is the one that + // raises SIGXFSZ (ignored) and returns `Err(EFBIG)`. The append must + // still report `Ok`, and the store must hold exactly the record that + // landed — not zero (treated as a real failure), not duplicated, and + // genuinely missing its `\n` rather than having silently written in + // full (the assertion below on the file's own last byte is what tells + // those two apart). A first, ordinary append before the limit is + // installed puts this at a non-zero `start` (`TailState::Clean` with an + // existing record ahead of it), not only the empty-file case. + #[cfg(unix)] + #[test] + fn append_tolerates_a_short_write_from_an_fsize_limit() -> Result<()> { + const SHORT_WRITE_CHILD_ENV: &str = "ROCM_CORE_TEST_JSONL_SHORT_WRITE_CHILD"; + + if let Some(data_dir) = std::env::var_os(SHORT_WRITE_CHILD_ENV) { + let data_dir = data_dir.to_string_lossy().into_owned(); + let paths = AppPaths { + config_dir: Path::new(&data_dir).join("config"), + data_dir: Path::new(&data_dir).to_path_buf(), + cache_dir: Path::new(&data_dir).join("cache"), + }; + append_audit_event(&paths, &audit_record("first"))?; + let record = audit_record("short-write"); + let mut line = serde_json::to_vec(&record).context("serialize audit record")?; + line.push(b'\n'); + let existing_len = fs::metadata(paths.audit_events_path()) + .context("stat seeded store")? + .len(); + with_lowered_fsize_limit(existing_len + line.len() as u64 - 1, || { + append_audit_event(&paths, &record) + })?; + return Ok(()); + } + + let (root, paths) = temp_app_paths("jsonl-short-write"); + paths.ensure()?; + let exe = std::env::current_exe().context("current test executable")?; + let output = std::process::Command::new(&exe) + .args([ + "automation::tests::append_tolerates_a_short_write_from_an_fsize_limit", + "--exact", + "--quiet", + "--nocapture", + ]) + .env(SHORT_WRITE_CHILD_ENV, paths.data_dir.display().to_string()) + .output() + .context("spawn short-write child")?; + let text = fs::read_to_string(paths.audit_events_path()); + fs::remove_dir_all(root).ok(); + assert!( + output.status.success(), + "child process failed: {:?}\nstdout: {}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let text = text?; + assert!( + !text.ends_with('\n'), + "the short write must have genuinely landed one byte short of its own \ + trailing newline — a file ending in `\\n` here means the rlimit never \ + actually cut the write, and this test proved nothing: {text:?}" + ); + let records: Vec = text + .lines() + .map(|line| { + serde_json::from_str(line) + .unwrap_or_else(|error| panic!("not a complete record: {error}: {line:?}")) + }) + .collect(); + assert_eq!( + audit_messages(&records), + ["first", "short-write"], + "the write that failed but landed must not be duplicated or dropped: {text:?}" + ); + Ok(()) + } + + // Sibling of the test above, for the other side of the same glue: a + // write that fails and lands *nothing* (the limit leaves no room even + // for the first byte) must surface as a genuine `Err`, not be folded + // into a false `Ok` the way a landed-but-incomplete write is. + // `landed_despite_the_error` itself already has a direct unit test for + // "nothing landing is not a complete record" — what has no test without + // this one is the `match` in `append_or_tolerate_a_missing_newline` that + // acts on that verdict: a mutation collapsing its `Ok(landed) if + // landed_despite_the_error(..) => Ok(())` / `Ok(_) => Err(..)` arms into + // an unconditional `Ok(())` would pass every other test in this file. + #[cfg(unix)] + #[test] + fn append_reports_a_real_failure_when_nothing_landed() -> Result<()> { + const NOTHING_LANDS_CHILD_ENV: &str = "ROCM_CORE_TEST_JSONL_NOTHING_LANDS_CHILD"; + + if let Some(data_dir) = std::env::var_os(NOTHING_LANDS_CHILD_ENV) { + let data_dir = data_dir.to_string_lossy().into_owned(); + let paths = AppPaths { + config_dir: Path::new(&data_dir).join("config"), + data_dir: Path::new(&data_dir).to_path_buf(), + cache_dir: Path::new(&data_dir).join("cache"), + }; + append_audit_event(&paths, &audit_record("first"))?; + let existing_len = fs::metadata(paths.audit_events_path()) + .context("stat seeded store")? + .len(); + // Zero slack: the write offset already sits at the limit, so + // `write(2)`'s very first call returns `Err(EFBIG)` having + // written nothing at all — not even a partial record. + let result = with_lowered_fsize_limit(existing_len, || { + append_audit_event(&paths, &audit_record("lost")) + }); + // Report the child's own verdict on stdout for the parent to + // read, rather than via exit status: an `Err` here is the + // *correct* outcome for this test, and letting it propagate as + // `?` would make the child process itself look like a failure + // for the wrong reason. The message itself (not just ok/err) is + // what tells a genuine write failure apart from some unrelated + // error (e.g. a lock or `paths.ensure()` failure) that would + // also make `is_err()` true without this test having proven + // anything. + match result { + Ok(()) => println!("ok"), + Err(error) => println!("err: {error:#}"), + } + return Ok(()); + } + + let (root, paths) = temp_app_paths("jsonl-nothing-lands"); + paths.ensure()?; + let exe = std::env::current_exe().context("current test executable")?; + let output = std::process::Command::new(&exe) + .args([ + "automation::tests::append_reports_a_real_failure_when_nothing_landed", + "--exact", + "--quiet", + "--nocapture", + ]) + .env( + NOTHING_LANDS_CHILD_ENV, + paths.data_dir.display().to_string(), + ) + .output() + .context("spawn nothing-lands child")?; + let text = fs::read_to_string(paths.audit_events_path()); + fs::remove_dir_all(root).ok(); + assert!( + output.status.success(), + "child process failed: {:?}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("err: failed to append"), + "a write that lands nothing must be reported as a real failure from the append \ + itself (not folded into a false success, and not some unrelated error): \ + child stdout was {stdout:?}" + ); + assert_eq!( + audit_messages(&[serde_json::from_str(text?.trim()).context("parse seeded record")?]), + ["first"], + "the store must be left exactly as the seed append wrote it — no partial or \ + lost record from the write that failed with nothing landing" + ); + Ok(()) + } + // A proposal-status update rewrites the whole proposals store, so it must // neither drop proposals appended while it runs nor be undone by them. #[test] @@ -1431,6 +1807,45 @@ mod tests { Ok(()) } + // Nothing in this module's own writers shrinks a store, but an + // out-of-band edit might, and load_recent_jsonl_records is supposed to + // retry from the new length rather than fail. Driving that needs the + // shrink to land strictly between two of ReverseLines's own chunked + // reads (see the `test_hooks` module doc) — which in turn + // needs a scan that takes more than one physical read in the first + // place, hence the pathologically long first record here (same + // technique as append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly + // below, just complete JSON rather than torn). + #[test] + fn load_recent_retries_past_a_shrink_mid_scan_and_succeeds() -> Result<()> { + let (root, paths) = temp_app_paths("scan-retry-after-shrink"); + let seed = format!("{}\n", audit_line("seed")); + let giant = audit_line(&"x".repeat(200_000)); + seed_audit_store(&paths, &format!("{seed}{giant}"))?; + let path = paths.audit_events_path(); + let seed_len = seed.len() as u64; + let hook_path = path.clone(); + test_hooks::set_before_scan_read( + &path, + 1, + std::sync::Arc::new(move || { + fs::OpenOptions::new() + .write(true) + .open(&hook_path) + .and_then(|f| f.set_len(seed_len)) + .expect("truncate mid-scan"); + }), + ); + let recent = load_recent_audit_events(&paths, 5); + fs::remove_dir_all(root).ok(); + assert_eq!( + audit_messages(&recent?), + ["seed"], + "a shrink strictly between two chunked reads must be retried, not failed" + ); + Ok(()) + } + #[test] fn reverse_lines_refuses_to_splice_across_a_mid_scan_shrink() -> Result<()> { let (root, _paths) = temp_app_paths("reverse-lines-shrink"); @@ -1518,6 +1933,26 @@ mod tests { Ok(()) } + // A whitespace-only unterminated tail is not valid JSON either, so + // without `is_blank` in `inspect_tail`'s check it would be classified + // Torn and cut by the next append's whole-store rewrite instead of being + // kept and terminated in place, as TailState::ValidUnterminated's own + // doc comment documents it should be. + #[test] + fn append_terminates_a_blank_unterminated_last_line_without_cutting_it() -> Result<()> { + let (root, paths) = temp_app_paths("append-blank-unterminated"); + seed_audit_store(&paths, &format!("{}\n ", audit_line("a")))?; + append_audit_event(&paths, &audit_record("b"))?; + let text = fs::read_to_string(paths.audit_events_path()); + fs::remove_dir_all(root).ok(); + assert_eq!( + text?, + format!("{}\n \n{}\n", audit_line("a"), audit_line("b")), + "a blank unterminated tail must be kept and terminated, not cut as torn debris" + ); + Ok(()) + } + #[test] fn append_leaves_a_torn_line_longer_than_one_scan_chunk_cut_cleanly() -> Result<()> { let (root, paths) = temp_app_paths("append-long-torn-tail"); From 35f36382764428ff38760238bf323e2de636817c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 27 Oct 2026 15:46:08 +0000 Subject: [PATCH 6/8] ROCMAI-721: add a scenario for rocm automations tolerating a torn audit tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reviewer on #738 pointed out that naming the existing unit tests (readers_skip_an_unterminated_last_line_that_does_not_parse, load_recent_stops_before_corrupt_lines_older_than_the_window) doesn't discharge AGENTS.md §3's scenario requirement for user-observable CLI behavior — those tests prove the internal helper, not the command. rocm automations is the production reader of the audit store specifically (confirmed: outside test code, render_automations_text is the only caller of load_recent_audit_events in the workspace), and it's a genuinely reachable, user-facing command — so this is the right level to pin the torn-tail-skip half of the behavior at. The corrupt-line-outside-the-window half stays a named unit test, since rocm automations uses a fixed internal limit rather than a user-supplied one, so there's no CLI-level lever to place a corrupt line on one side of the window or the other. Signed-off-by: Jussi Elo --- .../e2e-cucumber/features/automations.feature | 37 +++++++++-- .../tests/e2e/automations_steps.rs | 63 +++++++++++++++++-- 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index 2718a4713c..a195588c0b 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -4,10 +4,11 @@ Feature: Automation watchers # automation watcher in the CLI config and prints a confirmation; bare `rocm # automations` lists the most recent review requests (any status — only the # per-entry approve/reject controls are gated on "pending"). This feature - # pins the enable/disable/mode-confirmation slice plus the proposal-history - # listing's unit formatting; the watcher/daemon trigger behaviour itself is - # covered elsewhere. Config-only (no GPU, no network), so it runs on the - # mock lane every PR. + # pins the enable/disable/mode-confirmation slice, the proposal-history + # listing's unit formatting, and the bare listing's tolerance of a torn + # audit-log tail; the watcher/daemon trigger behaviour itself is covered + # elsewhere. Config-only (no GPU, no network), so it runs on the mock lane + # every PR. @id:automations-enable-confirms-mode Scenario: automations-01 - Enabling a watcher confirms its mode @@ -54,3 +55,31 @@ Feature: Automation watchers Given a pending artifact-download proposal approved for a size just under 1 MiB When the user lists automations Then the CLI reports the approved size in binary units + + # Readers of the audit/events/proposals stores take no lock, so one can + # catch a write mid-append; the cut-off last line that leaves is skipped + # instead of failing the whole read. + # `render_automations_text` already swallows any read error from these + # stores via `.unwrap_or_default()`, so `rocm automations` never exits + # non-zero either way — what this scenario actually pins is that the clean + # record ahead of the torn one still shows up, rather than the torn line's + # read error wiping out the whole audit section the way it would without + # this skip. + # `render_automations_text` is the production reader of the audit store + # specifically (the one this scenario seeds). The two places a person + # reads its raw text directly are the `rocm automations` CLI subcommand + # itself, and the TUI's automations-manager screen, which shells out to + # `rocm automations list` as a subprocess — so that's where the audit + # store's tolerance is directly user-observable. It's also reachable + # through other paths that don't show the raw text the same way — + # `rocm chat`'s and the TUI's slash-command `rocm_command` tool in-process + # (the model sees it, or the TUI shrinks it to field counts first), and + # rocmd's own MCP `rocm_command` tool via a further `rocm automations list` + # subprocess — this isn't meant as an exhaustive map of every caller, just + # context for why the CLI subcommand is the right level to pin this at. + @id:automations-tolerates-torn-audit-tail + Scenario: automations-06 - Listing automations tolerates a torn audit-log tail + Given a fresh CLI configuration + And the audit event log ends in a torn, unterminated line + When the user lists automations + Then the CLI still lists the record before the torn tail diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index a553cd6b37..3fad6b51ca 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -12,12 +12,13 @@ //! first plants an automation runtime-state marking the daemon already //! running under THIS test process's (live) pid, so the CLI's double-spawn //! guard skips the spawn. Contracts verified against the running Linux binary -//! (EAI-8072). Scoped to the enable/disable/mode slice plus the -//! proposal-history listing's unit formatting; the watcher/daemon trigger -//! behaviour itself is covered separately. +//! (EAI-8072). Scoped to the enable/disable/mode slice, the proposal-history +//! listing's unit formatting, and the bare listing's tolerance of a torn +//! audit-log tail; the watcher/daemon trigger behaviour itself is covered +//! separately. use cucumber::{given, then, when}; -use rocm_core::{AutomationProposalRecord, append_automation_proposal}; +use rocm_core::{AuditEventRecord, AutomationProposalRecord, append_automation_proposal}; use crate::E2eWorld; @@ -108,6 +109,41 @@ async fn pending_artifact_download_proposal(world: &mut E2eWorld) { .expect("failed to write automation proposal fixture"); } +/// Seed the audit-event store with one clean record followed by a cut-off +/// JSON fragment with no trailing newline — the shape a writer killed +/// mid-append leaves, and what a lock-free reader (`render_automations_text`, +/// reached by `rocm automations`, the `rocm_command` assistant tool shared +/// by `rocm chat` and the TUI's slash commands, and the TUI's own +/// automations-manager screen alike) must skip rather than fail the whole +/// read on. See `scan_recent_jsonl_records`/`is_torn_record` in `rocm-core` +/// (the read path this exercises — not `inspect_tail`, which is the write +/// path's own, separate torn-tail check). +#[given("the audit event log ends in a torn, unterminated line")] +async fn torn_audit_tail(world: &mut E2eWorld) { + let paths = world.app_paths(); + paths.ensure().expect("failed to create app dirs"); + let clean = AuditEventRecord { + at_unix_ms: 1_700_000_000_000, + source: "e2e".to_owned(), + category: "concurrency".to_owned(), + actor: "writer".to_owned(), + level: "info".to_owned(), + action: "append".to_owned(), + message: "clean".to_owned(), + watcher_id: None, + service_id: None, + }; + let clean_line = serde_json::to_string(&clean).expect("serialize audit record"); + // Just the first 20 bytes of the same record: short enough that the cut + // falls well before the `message` field, so what it says doesn't matter. + let torn_fragment = &clean_line[..20]; + std::fs::write( + paths.audit_events_path(), + format!("{clean_line}\n{torn_fragment}"), + ) + .expect("failed to seed a torn audit-event tail"); +} + #[given("an enabled automation watcher")] async fn enabled_watcher(world: &mut E2eWorld) { suppress_daemon_spawn(world); @@ -166,6 +202,25 @@ async fn enable_unknown(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── +#[then("the CLI still lists the record before the torn tail")] +async fn still_lists_record_before_torn_tail(world: &mut E2eWorld) { + // rc == 0 alone wouldn't prove much here: render_automations_text reads + // every store through `.unwrap_or_default()`, so even a hard read error + // would render as an empty section rather than a nonzero exit. The count + // below is the real assertion: `audit_event_plain_summary` renders every + // audit event's category to one of a handful of fixed strings (never the + // raw message), so the fixture's "concurrency" category falls through to + // the generic one; exactly one occurrence confirms the one clean record + // before the torn tail was actually read and rendered, not silently + // dropped along with it (zero) or duplicated (more than one). + let out = ok_output(world); + let activity_lines = out.matches("Background activity was recorded.").count(); + assert_eq!( + activity_lines, 1, + "expected exactly the one valid record before the torn tail to be listed, got:\n{out}" + ); +} + #[then(regex = r"^the CLI confirms the watcher is enabled in (observe|propose) mode$")] async fn confirm_enabled(world: &mut E2eWorld, mode: String) { let out = ok_output(world); From a0bd63547a25ccd41e095dccc1c9b41e757682e6 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 28 Oct 2026 11:12:40 +0000 Subject: [PATCH 7/8] ROCMAI-721: fix the two gaps this repo's automated reviewer found pr-review-watcher flagged two blocking issues on #738 after the prior review-response commits landed: - append_jsonl_record's doc claimed "seeing Err here means this specific record did not land" with one stated exception, but missed a second: if the write itself fails and the follow-up check of what actually landed also fails (a second I/O fault on the same unhealthy disk), the function reports the original write failure without knowing whether the record landed anyway. A caller retrying on that Err could duplicate it. Narrowed both append_jsonl_record's and append_or_tolerate_a_missing_newline's doc comments to state this explicitly, rather than plumbing a distinguishable error type through the whole call chain for a case no current caller's retry behavior actually exercises. - parse_jsonl_records only tolerates an unparseable line that is both unterminated and the store's last line. No existing test placed a corrupt line anywhere else, so removing the position check (verified by mutation) left every test green while silently letting update_automation_proposal_status's whole-file rewrite drop any corrupted line anywhere in the store. Added a_corrupt_line_before_the_last_is_reported_not_dropped, which seeds a corrupt line in the middle of a proposals store and asserts both the load and the status-update refuse, with the file left untouched. Also addressed, non-blocking: write_atomic's doc now describes the sharper same-process/same-millisecond temp-path collision (mixed bytes, a losing rename that fails outright) rather than just "loses one update"; reads_racing_appends_never_fail renamed to concurrent_reads_and_appends_never_error with a comment that states plainly what it can and cannot catch, instead of implying it pins the race itself; the torn-audit-tail scenario's rationale, previously duplicated near-verbatim across the feature file and two step comments, now lives once in the feature file; and prune_managed_service_records's "No nested acquire" comment now states the invariant directly instead of hand-maintaining an exhaustive list of lock call sites that kept going stale across review rounds. Every fix here went through its own fresh-review cycle: two early drafts had real bugs (a test fixture that didn't isolate the guard it meant to test, and a doc claim misattributing which function's return releases the launch lock) caught and corrected before landing. Signed-off-by: Jussi Elo --- apps/rocm/src/main.rs | 24 ++--- crates/rocm-core/src/automation.rs | 88 ++++++++++++++++--- crates/rocm-core/src/lib.rs | 10 +++ .../tests/e2e/automations_steps.rs | 28 +++--- 4 files changed, 111 insertions(+), 39 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 115cbb0b2e..33de62dce3 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -4564,16 +4564,20 @@ fn apply_service_prune_plan( /// run — the same kind of disagreement the paragraph above declines to ship. /// /// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so -/// a second acquire on a path already held by this process would hang forever. -/// Nothing under either phase acquires this path: the only other -/// `FileLock::acquire` calls in the tree are `ensure_background_helper_running_quiet`, -/// the managed-uv bootstrap in `rocm_core::uv`, and the per-store locks -/// `rocm_core`'s automation module takes around each audit, event and proposal -/// write — each on a different lock file — and none is reachable from here: -/// `rocm services prune` records its audit event only after this returns and -/// the lock is released. In the other -/// direction `serve` holds this lock but never runs prune, in-process or as a -/// subprocess. +/// a second acquire on this same lock path while this process already holds +/// it would hang forever. Nothing reachable from either phase re-acquires +/// `managed_launch_lock_path`: the only other production call site on this +/// same path is `select_gpu_indices_under_launch_lock`, which is only ever +/// reached from `serve`, and `serve` never runs prune in-process or as a +/// subprocess (test code acquires this path directly too, but that is test +/// setup, not something reachable from this function's own call graph). +/// +/// Separately, prune does cross a *different* lock — recording its audit +/// event takes that store's own `jsonl_lock_path` — but that call +/// (`record_cli_audit_event` in the `services` dispatch) happens only after +/// *this* function has already returned and `_launch_lock` has dropped, so +/// the two locks are never held at once and their acquisition order cannot +/// matter. fn prune_managed_service_records( paths: &AppPaths, hours: u64, diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index be6a3f22cb..28809f96e7 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -251,13 +251,22 @@ const JSONL_SCAN_CHUNK: u64 = 64 * 1024; /// /// A write that fails partway leaves whatever it already wrote as a new torn /// tail — the same debris a killed writer leaves — rather than rolling it -/// back: seeing `Err` here means this specific record did not land (see -/// [`append_or_tolerate_a_missing_newline`] for the one exception, where it -/// did land despite the error), and the debris itself is discarded by the -/// *next* append rather than being cut first as a separate step — see -/// [`inspect_tail`] and [`TailState::Torn`]. A reader already skips the -/// debris in the meantime. Nothing in this function ever truncates the live -/// file, so a lock-free reader never observes it shrink. +/// back, and the debris itself is discarded by the *next* append rather than +/// being cut first as a separate step — see [`inspect_tail`] and +/// [`TailState::Torn`]. A reader already skips the debris in the meantime. +/// Nothing in this function ever truncates the live file, so a lock-free +/// reader never observes it shrink. +/// +/// [`append_or_tolerate_a_missing_newline`] checks whether a failed write +/// landed anyway before reporting it, so a write that fails but still lands +/// a complete record is reported as `Ok`, not `Err`. Seeing `Err` here +/// usually does mean the record did not land — with one case where that's +/// not knowable rather than false: if the write itself fails *and* the +/// follow-up check of what actually landed also fails (a second I/O fault +/// on the same unhealthy disk, say), the original write failure is reported +/// without knowing whether the record landed anyway. A caller that retries +/// on `Err` in that specific, low-likelihood case could write a duplicate +/// the store has no way to detect or remove. fn append_jsonl_record(path: &Path, record: &T, context_label: &str) -> Result<()> { let mut line = serde_json::to_vec(record) .with_context(|| format!("failed to serialize {context_label}"))?; @@ -317,6 +326,11 @@ fn append_jsonl_record(path: &Path, record: &T, context_label: &st /// a failure would tell a caller that retries on error to write the same /// record again, leaving a duplicate the store has no way to detect or /// remove — so this checks what actually landed before deciding. +/// +/// That check can itself fail (stat or read-back hits its own I/O error), +/// in which case this reports the original write failure without knowing +/// whether the record landed anyway — the one case where `Err` here does +/// not guarantee the record is absent (see [`append_jsonl_record`]'s doc). fn append_or_tolerate_a_missing_newline( path: &Path, file: &fs::File, @@ -1583,13 +1597,18 @@ mod tests { Ok(()) } - // Readers take no lock, so they can catch an append mid-write; that must - // never surface as a parse error from `rocm status` or the dashboard. - // This is a smoke test: on Linux a small append is rarely seen half-done, - // so the skip itself is pinned deterministically by - // `readers_skip_an_unterminated_last_line_that_does_not_parse`. + // Smoke test, not a race-pinning one: these stores stay well under one + // scan chunk, so on Linux a small append is essentially never actually + // observed half-written here, and this can mostly only fail via a + // propagated read/write error or the final count assertion — not by + // actually exercising the torn-line skip under real contention. That + // skip is pinned deterministically instead by + // `readers_skip_an_unterminated_last_line_that_does_not_parse`. What + // this test is still worth keeping for: a reader that started failing + // outright under concurrent load (rather than just reading stale data, + // which this can't detect either) would still show up here. #[test] - fn reads_racing_appends_never_fail() -> Result<()> { + fn concurrent_reads_and_appends_never_error() -> Result<()> { const APPENDS: usize = 200; let (root, paths) = temp_app_paths("jsonl-read-race"); paths.ensure()?; @@ -1787,6 +1806,49 @@ mod tests { Ok(()) } + // Mirrors the test above, but for the position half of + // parse_jsonl_records's guard rather than its shape half: a line that + // is not valid JSON (so `is_torn_record` is true for it, same as a + // genuine torn tail) but is NOT the store's last line must still be + // reported as an error, not silently skipped. Without the + // `index + 1 == line_count` half of that guard, this line would be + // indistinguishable from torn debris and silently dropped by + // update_automation_proposal_status's whole-file rewrite. + #[test] + fn a_corrupt_line_before_the_last_is_reported_not_dropped() -> Result<()> { + let (root, paths) = temp_app_paths("corrupt-line-not-last"); + paths.ensure()?; + let corrupt = &serde_json::to_string(&proposal_record("corrupt"))?[..10]; + // No trailing `\n` after "b": the buffer as a whole must be + // unterminated too, so the corrupt line is excluded from the + // tolerance window by the position check specifically + // (`index + 1 == line_count`) and not merely because the whole + // buffer happens to already be terminated. + let contents = format!( + "{}\n{corrupt}\n{}", + serde_json::to_string(&proposal_record("a"))?, + serde_json::to_string(&proposal_record("b"))? + ); + fs::write(paths.automation_proposals_path(), &contents)?; + let proposals = load_automation_proposals(&paths); + let updated = update_automation_proposal_status(&paths, "a", "approved"); + let on_disk = fs::read_to_string(paths.automation_proposals_path()); + fs::remove_dir_all(root).ok(); + assert!( + proposals.is_err(), + "a corrupt line that isn't the store's last line must be reported, not silently skipped" + ); + assert!( + updated.is_err(), + "the rewrite must refuse, not drop the line" + ); + assert_eq!( + on_disk?, contents, + "a refused rewrite must leave the file untouched" + ); + Ok(()) + } + // A lock-free reader that measured a stale, larger length — because the // store was edited out-of-band, not because a concurrent writer shrank // it — sees the file end early; that is not an error. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f2963bc520..cf56ff4fe6 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -358,6 +358,16 @@ pub fn unix_time_millis() -> u128 { /// cycles that overlap still lose one update (last rename wins). A caller /// whose new content depends on the old must hold a lock across the whole /// cycle, as the JSONL proposal store does (`automation::jsonl_lock_path`). +/// Two callers in the *same process* within the same millisecond are a +/// sharper case of this, not just a milder one: they generate the exact +/// same temp path below, so concurrent `fs::write`s truncating that one +/// path can publish mixed bytes, and the losing rename then fails outright +/// (its source file already gone) rather than merely losing an update — +/// this is the same-process collision +/// `automation::tests::concurrent_appends_from_many_threads_lose_no_records`'s +/// own comment describes ("two threads of one process even shared a +/// temp-file name") — fixed for its own callers by the per-store lock +/// `automation.rs` takes before ever calling into this function. /// - Durability: the bytes are not fsynced before the rename, so a power /// loss can still surface the new name with unflushed contents. /// - Replacing a destination another process holds open without delete diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 3fad6b51ca..b04d2dfd0f 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -111,13 +111,11 @@ async fn pending_artifact_download_proposal(world: &mut E2eWorld) { /// Seed the audit-event store with one clean record followed by a cut-off /// JSON fragment with no trailing newline — the shape a writer killed -/// mid-append leaves, and what a lock-free reader (`render_automations_text`, -/// reached by `rocm automations`, the `rocm_command` assistant tool shared -/// by `rocm chat` and the TUI's slash commands, and the TUI's own -/// automations-manager screen alike) must skip rather than fail the whole -/// read on. See `scan_recent_jsonl_records`/`is_torn_record` in `rocm-core` -/// (the read path this exercises — not `inspect_tail`, which is the write -/// path's own, separate torn-tail check). +/// mid-append leaves. See the scenario's own comment in `automations.feature` +/// for why this matters and who reads this store; see +/// `scan_recent_jsonl_records`/`is_torn_record` in `rocm-core` for the read +/// path this exercises (not `inspect_tail`, which is the write path's own, +/// separate torn-tail check). #[given("the audit event log ends in a torn, unterminated line")] async fn torn_audit_tail(world: &mut E2eWorld) { let paths = world.app_paths(); @@ -204,15 +202,13 @@ async fn enable_unknown(world: &mut E2eWorld) { #[then("the CLI still lists the record before the torn tail")] async fn still_lists_record_before_torn_tail(world: &mut E2eWorld) { - // rc == 0 alone wouldn't prove much here: render_automations_text reads - // every store through `.unwrap_or_default()`, so even a hard read error - // would render as an empty section rather than a nonzero exit. The count - // below is the real assertion: `audit_event_plain_summary` renders every - // audit event's category to one of a handful of fixed strings (never the - // raw message), so the fixture's "concurrency" category falls through to - // the generic one; exactly one occurrence confirms the one clean record - // before the torn tail was actually read and rendered, not silently - // dropped along with it (zero) or duplicated (more than one). + // The count, not rc == 0, is the real assertion here (see + // `automations.feature` for why). `audit_event_plain_summary` renders + // every audit event's category to one of a handful of fixed strings + // (never the raw message), so the fixture's "concurrency" category falls + // through to the generic one; exactly one occurrence confirms the one + // clean record before the torn tail was actually read and rendered, not + // silently dropped along with it (zero) or duplicated (more than one). let out = ok_output(world); let activity_lines = out.matches("Background activity was recorded.").count(); assert_eq!( From 40a59388f99bc21e0ceaa868207d6738c7cad8ad Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 28 Oct 2026 11:47:01 +0000 Subject: [PATCH 8/8] ROCMAI-721: dedupe the two thread-based concurrency tests' scaffolding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit juhovainio approved #738 but noted concurrent_appends_after_a_torn_tail_lose_no_records duplicates concurrent_appends_from_many_threads_lose_no_records's thread-spawn-and-barrier scaffolding almost verbatim, differing only in whether a torn tail is seeded first and what tags are expected in the result. Extracted the shared part into run_concurrent_thread_appends, which both tests now call, keeping only their own seed step (if any) and their own expected-tags assertion. Behavior is unchanged: same writer/append counts, same seeding, same assertions — verified by re-running both tests and by mutation (removing the per-store lock from the production append path still fails both the same way it did before this refactor). Signed-off-by: Jussi Elo --- crates/rocm-core/src/automation.rs | 68 ++++++++++++++---------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index 28809f96e7..fc7f727fd6 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -1146,25 +1146,26 @@ mod tests { } } - // Regression test for concurrent writers silently dropping each other's - // records: every append used to read the whole store, add one line in - // memory, and rename a rewritten copy over the original, so two writers - // that overlapped kept only the last rename (and two threads of one - // process even shared a temp-file name). - #[test] - fn concurrent_appends_from_many_threads_lose_no_records() -> Result<()> { - const WRITERS: usize = 8; - const APPENDS_PER_WRITER: usize = 25; - let (root, paths) = temp_app_paths("jsonl-concurrent-threads"); - paths.ensure()?; - let barrier = std::sync::Barrier::new(WRITERS); + /// Run `writers` threads, each appending `appends_per_writer` tagged + /// records (`t-`) to all three stores concurrently (a + /// `Barrier` lines them up so the appends actually overlap rather than + /// running one at a time), then return what each store actually holds + /// afterward. Shared by the two tests immediately below, which differ + /// only in what (if anything) was in the stores before this runs and in + /// what tags they expect alongside `t-`. + fn run_concurrent_thread_appends( + paths: &AppPaths, + writers: usize, + appends_per_writer: usize, + ) -> Result<[Vec; 3]> { + let barrier = std::sync::Barrier::new(writers); std::thread::scope(|scope| { - let handles: Vec<_> = (0..WRITERS) + let handles: Vec<_> = (0..writers) .map(|writer| { - let (paths, barrier) = (&paths, &barrier); + let (paths, barrier) = (paths, &barrier); scope.spawn(move || -> Result<()> { barrier.wait(); - for n in 0..APPENDS_PER_WRITER { + for n in 0..appends_per_writer { append_to_every_store(paths, &format!("t{writer}-{n}"))?; } Ok(()) @@ -1176,11 +1177,25 @@ mod tests { } Ok::<_, anyhow::Error>(()) })?; + tags_in_every_store(paths) + } + + // Regression test for concurrent writers silently dropping each other's + // records: every append used to read the whole store, add one line in + // memory, and rename a rewritten copy over the original, so two writers + // that overlapped kept only the last rename (and two threads of one + // process even shared a temp-file name). + #[test] + fn concurrent_appends_from_many_threads_lose_no_records() -> Result<()> { + const WRITERS: usize = 8; + const APPENDS_PER_WRITER: usize = 25; + let (root, paths) = temp_app_paths("jsonl-concurrent-threads"); + paths.ensure()?; + let stores = run_concurrent_thread_appends(&paths, WRITERS, APPENDS_PER_WRITER); let expected: Vec = (0..WRITERS) .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("t{writer}-{n}"))) .collect(); - let stores = tags_in_every_store(&paths); fs::remove_dir_all(root).ok(); assert_every_store_holds_exactly(stores?, &expected); Ok(()) @@ -1227,32 +1242,13 @@ mod tests { const APPENDS_PER_WRITER: usize = 25; let (root, paths) = temp_app_paths("jsonl-torn-tail-concurrent"); seed_torn_tail_in_every_store(&paths)?; - let barrier = std::sync::Barrier::new(WRITERS); - std::thread::scope(|scope| { - let handles: Vec<_> = (0..WRITERS) - .map(|writer| { - let (paths, barrier) = (&paths, &barrier); - scope.spawn(move || -> Result<()> { - barrier.wait(); - for n in 0..APPENDS_PER_WRITER { - append_to_every_store(paths, &format!("t{writer}-{n}"))?; - } - Ok(()) - }) - }) - .collect(); - for handle in handles { - handle.join().expect("writer thread panicked")?; - } - Ok::<_, anyhow::Error>(()) - })?; + let stores = run_concurrent_thread_appends(&paths, WRITERS, APPENDS_PER_WRITER); let mut expected: Vec = vec!["seed".to_owned()]; expected.extend( (0..WRITERS) .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("t{writer}-{n}"))), ); - let stores = tags_in_every_store(&paths); fs::remove_dir_all(root).ok(); assert_every_store_holds_exactly(stores?, &expected); Ok(())