diff --git a/Cargo.lock b/Cargo.lock index bcbd0022c1..f0f9741eba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3708,6 +3708,7 @@ dependencies = [ "rand 0.9.4", "regex", "rsa", + "same-file", "serde", "serde_json", "sha2", diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 79efcb2692..7cb967cef0 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -854,6 +854,23 @@ fn pending_stop_proposal_exists(paths: &AppPaths, service_id: &str) -> Result Result { + Ok(rocm_core::load_automation_proposals(paths)? + .into_iter() + .any(|proposal| { + proposal.status == "pending" + && proposal.watcher_id == "server-recover" + && proposal.service_id.as_deref() == Some(service_id) + && (proposal.action == "queue_restart_proposal" + || proposal.tool.as_deref() == Some("restart_server")) + })) +} + fn handle_cache_warm_event( paths: &AppPaths, mode: WatcherMode, @@ -1225,6 +1242,43 @@ fn handle_server_recover_event_with_record( Some(record.service_id.clone()), ), WatcherPolicyAction::QueueProposal => { + // An unreadable proposals store is reported and skipped, never + // propagated: the daemon loop exits on an error out of + // `evaluate_watchers`, so a store that stays corrupt would take it + // down on every recovery tick. + match pending_restart_proposal_exists(paths, &record.service_id) { + Ok(true) => { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "info", + "restart_proposal_already_pending", + &format!( + "managed service {} needs recovery ({recovery_reason_display}); a restart request awaiting review is already queued", + record.service_id, + ), + Some(record.service_id.clone()), + ); + } + Ok(false) => {} + Err(error) => { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "error", + "restart_proposal_check_failed", + &format!( + "managed service {} needs recovery ({recovery_reason_display}); \ + not queueing a restart request because the proposals store could \ + not be checked for one already pending: {error:#}", + record.service_id, + ), + Some(record.service_id.clone()), + ); + } + } let message = format!( "managed service {} needs recovery ({recovery_reason_display}); queueing restart proposal", record.service_id, @@ -2234,6 +2288,277 @@ mod tests { Ok(()) } + #[test] + fn server_recover_does_not_duplicate_pending_restart_proposals() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let events = rocm_core::load_recent_automation_events(&paths, 2)?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 10)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("restart_server")); + // The wording describes a proposal still awaiting review; the single + // pending proposal asserted here is what makes that true. + assert_eq!(proposals[0].status, "pending"); + assert!(events.iter().any(|event| { + event.action == "restart_proposal_already_pending" + && event.message.contains("a restart request awaiting review") + })); + Ok(()) + } + + #[test] + fn server_recover_dedupe_sees_a_pending_restart_older_than_the_recent_window() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe-old"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + // Enough newer proposals from other watchers that the restart proposal + // is no longer among the 100 most recent. + for index in 0..101 { + queue_proposal_with_arguments( + &paths, + "gpu-thermal-protect", + "queue_stop_server_proposal", + "Stop another server", + "unrelated proposal", + Some(format!("svc-other-{index}")), + json!({}), + )?; + } + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let restarts = rocm_core::load_automation_proposals(&paths)? + .into_iter() + .filter(|proposal| proposal.tool.as_deref() == Some("restart_server")) + .count(); + fs::remove_dir_all(root).ok(); + + assert_eq!(restarts, 1); + Ok(()) + } + + #[test] + fn server_recover_dedupe_ignores_proposals_that_do_not_block_a_restart() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe-filters"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + // Each of these matches the restart check on all but one field, so + // none of them may suppress a new restart for `svc-unhealthy`. + for (watcher_id, service_id, status) in [ + ("server-recover", "svc-unhealthy", "approved"), + ("server-recover", "svc-other", "pending"), + ("another-watcher", "svc-unhealthy", "pending"), + ] { + append_automation_proposal( + &paths, + &AutomationProposalRecord { + at_unix_ms: unix_time_millis(), + proposal_id: String::new(), + watcher_id: watcher_id.to_owned(), + action: "queue_restart_proposal".to_owned(), + title: "Restart".to_owned(), + message: "unrelated".to_owned(), + status: status.to_owned(), + service_id: Some(service_id.to_owned()), + tool: Some("restart_server".to_owned()), + arguments: json!({}), + reviewed_at_unix_ms: None, + }, + )?; + } + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let queued = rocm_core::load_automation_proposals(&paths)? + .into_iter() + .filter(|proposal| { + proposal.watcher_id == "server-recover" + && proposal.status == "pending" + && proposal.service_id.as_deref() == Some("svc-unhealthy") + }) + .count(); + fs::remove_dir_all(root).ok(); + + assert_eq!(queued, 1); + Ok(()) + } + + #[test] + fn server_recover_reports_an_unreadable_proposals_store_instead_of_failing() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-corrupt-store"); + paths.ensure()?; + fs::write(paths.automation_proposals_path(), "not a proposal\n")?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + let outcome = handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + ); + let events = rocm_core::load_recent_automation_events(&paths, 1)?; + let stored = fs::read_to_string(paths.automation_proposals_path())?; + fs::remove_dir_all(root).ok(); + + // An error out of a watcher ends the daemon loop, so a corrupt store + // must surface as an event and a skipped tick, not as `Err`. + outcome?; + assert_eq!(stored, "not a proposal\n", "nothing may be queued"); + assert!(events.iter().any(|event| { + event.level == "error" && event.action == "restart_proposal_check_failed" + })); + Ok(()) + } + #[test] fn local_webhook_gpu_metrics_event_uses_existing_read_only_policy() -> Result<()> { let (root, paths) = temp_app_paths("local-webhook-gpu-metrics"); diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index 8dc0d950b2..b3a7f6a7e7 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -25,6 +25,9 @@ libc.workspace = true rand.workspace = true regex = "1" rsa.workspace = true +# File identity (inode / Windows file index), so the lock-free JSONL readers in +# `automation.rs` can tell when a store rotation renamed a file under them. +same-file = "1" serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index fc7f727fd6..f1a286ebd6 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -22,7 +22,18 @@ //! the whole store via `write_atomic` too. Nothing in this module ever //! shrinks a store in place. Readers take no lock, and an unterminated last //! line that is not valid JSON — an append caught mid-write — is skipped -//! rather than reported as corruption. The stores are not rotated or capped. +//! rather than reported as corruption. +//! +//! The events and audit stores are additionally capped by size-based rotation +//! (see [`EVENT_LOG_ROTATE_BYTES`]): the append that finds the live file at +//! the cap renames it whole to `.1` first, under the same lock — +//! before any torn tail in it would otherwise be repaired, so an at-cap store +//! is never paid for with a `write_atomic` rewrite only to be rotated away +//! immediately after — and the recent-record readers span the rotated +//! generations. The proposals store is never rotated, because a pending +//! proposal must stay findable by id until it is reviewed; instead the status +//! rewrite compacts it, dropping all but the newest +//! [`REVIEWED_PROPOSALS_RETAINED`] reviewed proposals. //! //! `AutomationTriggerEvent` is also defined here, but this module persists //! none of it: it is the in-memory trigger type passed between `rocmd`'s @@ -30,11 +41,14 @@ //! `apps/rocmd/src/webhook.rs`), never serialized. use anyhow::{Context, Result, bail}; +use same_file::Handle; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; use std::fs; -use std::io::{Read, Seek, SeekFrom, Write}; +use std::io::{ErrorKind, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::sync::{Mutex, PoisonError}; use std::time::{SystemTime, UNIX_EPOCH}; use crate::AppPaths; @@ -228,10 +242,13 @@ pub struct AuditEventRecord { /// /// Appends and the proposal-status rewrite both take it. The proposal /// rewrite needs it for its read-modify-write; an append needs it both to -/// serialize against other appends and because it can itself become a -/// rewrite — a torn tail is repaired by renaming a new file over the store -/// (see [`TailState::Torn`]) — and an append that had already opened the old -/// file would otherwise land in the replaced copy and vanish. +/// serialize against other appends and because it can itself replace or +/// rename the live file out from under a concurrent writer — a torn tail is +/// repaired by renaming a new file over the store (see [`TailState::Torn`]), +/// and a rotating events or audit store is renamed aside to `.1` (see +/// [`rotate_jsonl_generations`]) — and an append that had already opened the +/// old file would otherwise land in the replaced or rotated-away copy and +/// vanish. fn jsonl_lock_path(path: &Path) -> PathBuf { let mut name = path.file_name().unwrap_or_default().to_os_string(); name.push(".lock"); @@ -241,6 +258,102 @@ fn jsonl_lock_path(path: &Path) -> PathBuf { /// How many bytes the backward scans below read per step. const JSONL_SCAN_CHUNK: u64 = 64 * 1024; +/// Size at which the automation events store and the audit store are rotated: +/// the next append renames the live file to `.1` and starts a new one. +/// +/// With the `gpu-metrics` watcher enabled, the busiest writer, `rocmd` adds a +/// record a minute to each store — about 600 KB a day to the audit log — so +/// one generation holds roughly two weeks of history; without it, months. +/// Small enough that a rotated file can still be opened in an editor or read +/// whole by a script. +pub const EVENT_LOG_ROTATE_BYTES: u64 = 8 * 1024 * 1024; + +/// Rotated generations kept per store besides the live file. +/// +/// `.1` is the newest, `.4` the oldest; the oldest is dropped by +/// the next rotation. Caps each store at about 40 MiB — about two months of +/// history at the `gpu-metrics` rate — where it previously grew without bound. +/// A store that grew past the cap before rotation existed is rotated whole, so +/// it only falls under the cap once that file is itself dropped. +pub const EVENT_LOG_ROTATED_GENERATIONS: usize = 4; + +/// Reviewed proposals kept by the proposals store's compaction. +/// +/// "Reviewed" is any status but `pending` (approved, rejected, ...); the most +/// recently reviewed are kept, and pending proposals always are. Reviewed +/// proposals only come from the status rewrite that compacts, so this bounds +/// them exactly. +pub const REVIEWED_PROPOSALS_RETAINED: usize = 200; + +/// Size-based rotation policy for a JSONL store. +#[derive(Debug, Clone, Copy)] +struct JsonlRotation { + /// Rotate before an append once the live file is at least this long. + max_bytes: u64, + /// Rotated generations to keep besides the live file; at least 1. + generations: usize, +} + +/// The policy for the automation events store and the audit store. +const EVENT_LOG_ROTATION: JsonlRotation = JsonlRotation { + max_bytes: EVENT_LOG_ROTATE_BYTES, + generations: EVENT_LOG_ROTATED_GENERATIONS, +}; + +/// Path of generation `generation` of the store at `path`: the live file for +/// `0`, else `.` beside it (`events.jsonl.1`). +fn jsonl_generation_path(path: &Path, generation: usize) -> PathBuf { + if generation == 0 { + return path.to_path_buf(); + } + let mut name = path.file_name().unwrap_or_default().to_os_string(); + name.push(format!(".{generation}")); + path.with_file_name(name) +} + +/// Rotate the live file to `.1`, first shifting the generations in its way +/// up by one — `.{n-1}` over `.{n}` (dropping the oldest), …, `.1` to `.2` — +/// and leaving no live file for the caller's append to create. The caller must +/// hold the store's [`jsonl_lock_path`] lock, so no append can land in a file +/// being renamed. +/// +/// Only the unbroken run of generations from `.1` up is shifted: a missing +/// generation is a free slot, so nothing older than it has to move, and +/// nothing is dropped. That matters after a rotation that stopped part-way +/// (shifted `.1` away, then failed to rename the live file, leaving a gap at +/// `.1`): the next rotation then only fills the gap, instead of shifting — and +/// dropping — one more generation each time it runs. +/// +/// Renames newest-last, so at every instant the files present, read live +/// first, are still in order with no file under two names; a lock-free reader +/// relies on that (see [`open_jsonl_generations`]). Stops at the first failed +/// rename: going on could rename a file over one that was never shifted away. +fn rotate_jsonl_generations(path: &Path, generations: usize) -> Result<()> { + let mut run = 0; + while run < generations && fs::symlink_metadata(jsonl_generation_path(path, run + 1)).is_ok() { + run += 1; + } + // When every generation exists, the oldest is renamed over, not moved. + for generation in (1..=run.min(generations - 1)).rev() { + let from = jsonl_generation_path(path, generation); + let to = jsonl_generation_path(path, generation + 1); + fs::rename(&from, &to) + .with_context(|| format!("failed to rotate {} to {}", from.display(), to.display()))?; + } + let first = jsonl_generation_path(path, 1); + fs::rename(path, &first) + .with_context(|| format!("failed to rotate {} to {}", path.display(), first.display())) +} + +fn open_jsonl_for_append(path: &Path) -> Result { + fs::OpenOptions::new() + .read(true) + .append(true) + .create(true) + .open(path) + .with_context(|| format!("failed to open {}", path.display())) +} + /// Append one JSONL `record` to `path`, creating the file if needed. /// /// Holds the store's [`jsonl_lock_path`] lock. The common case — the store @@ -267,18 +380,44 @@ const JSONL_SCAN_CHUNK: u64 = 64 * 1024; /// without knowing whether the record landed anyway. A caller that retries /// on `Err` in that specific, low-likelihood case could write a duplicate /// the store has no way to detect or remove. -fn append_jsonl_record(path: &Path, record: &T, context_label: &str) -> Result<()> { +/// +/// With a `rotation` policy, a live file already at `max_bytes` is rotated +/// aside (see [`rotate_jsonl_generations`]) *before* [`inspect_tail`] runs — +/// whole, debris and all, rather than repairing a torn tail first and +/// rotating the repaired copy away a moment later. Rotation is best effort: +/// if it fails, the record is still appended to the unrotated file via the +/// normal `inspect_tail` path below, and the next append tries again — losing +/// a record to a failed rename would be worse than one oversized file. +fn append_jsonl_record( + path: &Path, + record: &T, + rotation: Option, + context_label: &str, +) -> Result<()> { let mut line = serde_json::to_vec(record) .with_context(|| format!("failed to serialize {context_label}"))?; line.push(b'\n'); let _lock = FileLock::acquire(jsonl_lock_path(path))?; - let file = fs::OpenOptions::new() - .read(true) - .append(true) - .create(true) - .open(path) - .with_context(|| format!("failed to open {}", path.display()))?; + let mut file = open_jsonl_for_append(path)?; + if let Some(rotation) = rotation { + let len = file + .metadata() + .with_context(|| format!("failed to stat {}", path.display()))? + .len(); + if len >= rotation.max_bytes { + // Close the live file before renaming it, so no write handle of + // ours is held across the rename and the reopen below gets the + // new file. + drop(file); + if let Err(error) = rotate_jsonl_generations(path, rotation.generations) + && let Some(warning) = rotation_failure_warning(path, &error) + { + eprintln!("{warning}"); + } + file = open_jsonl_for_append(path)?; + } + } match inspect_tail(path, &file)? { TailState::Torn(tail_start) => { // A torn tail is debris — crash or a failed write — that must be @@ -316,6 +455,21 @@ fn append_jsonl_record(path: &Path, record: &T, context_label: &st Ok(()) } +/// The warning to print that rotating `path` failed, or `None` after the first +/// one for that store in this process, so a rename that keeps failing (e.g. +/// another program holds the live file open without delete sharing on +/// Windows) is observable without a line on every single append while the +/// store stays stuck over its cap. Keyed by path so one store's failure does +/// not silence another's. +fn rotation_failure_warning(path: &Path, error: &anyhow::Error) -> Option { + static WARNED: Mutex> = Mutex::new(BTreeSet::new()); + WARNED + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(path.to_path_buf()) + .then(|| format!("warning: failed to rotate {}: {error:#}", path.display())) +} + /// Append-mode write of `line`, starting at offset `start`. /// /// A write can fail after landing every byte of `line` except its final @@ -492,14 +646,132 @@ fn parse_jsonl_records( Ok(records) } -/// Read the most recent `limit` JSONL records from `path`, in on-disk order. -/// Returns an empty vec when `path` doesn't exist yet or `limit` is `0`. +/// Read the most recent `limit` JSONL records from the store at `path` and +/// its first `generations` rotated generations (see +/// [`rotate_jsonl_generations`]), in on-disk order, oldest first. Returns an +/// empty vec when no generation exists yet or `limit` is `0`. /// -/// Scans backwards from the end of the file and stops once it has `limit` -/// records, so the cost tracks `limit`, not the size of the store; a corrupt -/// line older than those records is therefore not reported. As in -/// [`parse_jsonl_records`], an unterminated last line that is not valid JSON is -/// an append caught mid-write and is skipped. +/// Reads the live file first and moves on to `.1`, `.2`, … only while it is +/// still short of `limit`, so a read just after a rotation still returns the +/// records that were rotated aside. Each file is scanned backwards from its +/// end, stopping once it has `limit` records, so the cost tracks `limit`, not +/// the size of the store; a corrupt line older than those records is +/// therefore not reported. As in [`parse_jsonl_records`], an unterminated +/// last line that is not valid JSON is an append caught mid-write and is +/// skipped. +fn load_recent_jsonl_records( + path: &Path, + generations: usize, + limit: usize, + context_label: &str, +) -> Result> { + if limit == 0 { + return Ok(Vec::new()); + } + let mut newest_first = Vec::new(); + let mut found = 0; + for (generation_path, handle) in open_jsonl_generations(path, generations)? { + let records = load_recent_jsonl_records_from( + &generation_path, + handle.as_file(), + limit - found, + context_label, + )?; + found += records.len(); + newest_first.push(records); + if found >= limit { + break; + } + } + Ok(newest_first.into_iter().rev().flatten().collect()) +} + +/// Open the store's live file and its first `generations` rotated +/// generations, newest first, skipping any that do not exist — as one +/// consistent snapshot, so the readers can take no lock. +/// +/// A rotation can rename files between two of the opens: opening the live +/// file and then `.1` across one would open the same file twice (and read its +/// records twice), and opening `.1` and then the live file across one would +/// skip a generation. A torn-tail repair (see [`TailState::Torn`]) also +/// replaces the live file, by renaming a new one over it — not a rotation, +/// but the same hazard for this snapshot: a path opened before the repair +/// would still read the pre-repair file by handle, yet a path opened after +/// would see different content at the same name. So after opening them all, +/// every path is opened again and must still name the same file, or still be +/// missing; if any moved, the whole set is reopened — the identity check does +/// not need to tell a rotation from a repair apart, it only needs one retry +/// either way. A rotation only ever moves a file to a higher generation, and +/// a new live file (whether from rotation or repair) is a new file, so a path +/// naming the same file at both opens named it throughout: the handles are +/// what every path named at one instant between the two passes, which +/// [`rotate_jsonl_generations`] keeps in order with no file under two names. +/// The handles stay open, so a dropped generation is still readable and its +/// file identity cannot be reused meanwhile. +/// +/// Giving up after eight retries returns an error naming `path`, but some +/// callers (the dashboard's status view, among others) read recent records +/// through `.unwrap_or_default()` and would present that as an empty +/// activity list rather than surface it. Accepted: reaching the retry limit +/// needs the file to keep moving under every one of eight consecutive opens, +/// far more contention than a rotation once per `EVENT_LOG_ROTATE_BYTES` of +/// appends produces, and the very next read tries again from scratch. +fn open_jsonl_generations(path: &Path, generations: usize) -> Result> { + // A rotation happens once per `EVENT_LOG_ROTATE_BYTES` of appends, so + // even one retry is rare; running out means something else is renaming + // the files. + const ATTEMPTS: usize = 8; + let open = |path: &Path| -> Result> { + match fs::File::open(path) { + Ok(file) => Handle::from_file(file) + .map(Some) + .with_context(|| format!("failed to identify {}", path.display())), + // Windows cannot open a directory this way; like a missing file, + // it holds no records (a directory opened on Linux is skipped + // below). + Err(error) if error.kind() == ErrorKind::NotFound || path.is_dir() => Ok(None), + Err(error) => Err(error).with_context(|| format!("failed to open {}", path.display())), + } + }; + let paths: Vec = (0..=generations) + .map(|generation| jsonl_generation_path(path, generation)) + .collect(); + for _ in 0..ATTEMPTS { + let handles = paths + .iter() + .map(|path| open(path)) + .collect::>>()?; + // A single file cannot be read twice or skipped. + let mut stable = generations == 0; + if !stable { + stable = true; + for (path, handle) in paths.iter().zip(&handles) { + if open(path)?.as_ref() != handle.as_ref() { + stable = false; + break; + } + } + } + if !stable { + continue; + } + let mut files = Vec::new(); + for (path, handle) in paths.into_iter().zip(handles) { + let Some(handle) = handle else { continue }; + let metadata = handle + .as_file() + .metadata() + .with_context(|| format!("failed to stat {}", path.display()))?; + if metadata.is_file() { + files.push((path, handle)); + } + } + return Ok(files); + } + bail!("{} kept being rotated while it was read", path.display()) +} + +/// [`load_recent_jsonl_records`] over one file, `file`, opened from `path`. /// /// The scan takes no lock. A concurrent append only ever grows the file (see /// [`append_jsonl_record`]); nothing this module's own writers do shrinks it @@ -507,17 +779,16 @@ fn parse_jsonl_records( /// — by a person, not a concurrent writer — while a scan is in flight: a /// scan that fails while the file's length moved under it is retried from /// the new end, so that still never surfaces as an error. -fn load_recent_jsonl_records( +fn load_recent_jsonl_records_from( path: &Path, + file: &fs::File, limit: usize, context_label: &str, ) -> Result> { const ATTEMPTS: usize = 3; - if !path.is_file() || limit == 0 { + if limit == 0 { return Ok(Vec::new()); } - let file = - fs::File::open(path).with_context(|| format!("failed to open {}", path.display()))?; let file_len = || { file.metadata() .map(|metadata| metadata.len()) @@ -526,15 +797,15 @@ fn load_recent_jsonl_records( let mut attempt = 1; loop { let len = file_len()?; - match scan_recent_jsonl_records(path, &file, len, limit, context_label) { + match scan_recent_jsonl_records(path, file, len, limit, context_label) { Err(_) if attempt < ATTEMPTS && file_len()? != len => attempt += 1, outcome => return outcome, } } } -/// One backward scan of [`load_recent_jsonl_records`], over the first `len` -/// bytes of `file`. +/// One backward scan of [`load_recent_jsonl_records_from`], over the first +/// `len` bytes of `file`. fn scan_recent_jsonl_records( path: &Path, file: &fs::File, @@ -719,6 +990,7 @@ pub fn append_automation_event(paths: &AppPaths, event: &AutomationEventRecord) append_jsonl_record( &paths.automation_events_path(), event, + Some(EVENT_LOG_ROTATION), "automation event record", ) } @@ -732,16 +1004,24 @@ pub fn append_automation_proposal( if proposal.proposal_id.is_empty() { proposal.proposal_id = generate_proposal_id(&proposal.watcher_id); } + // Never rotated: a pending proposal must stay findable by id until it is + // reviewed. The status rewrite compacts the store instead. append_jsonl_record( &paths.automation_proposals_path(), &proposal, + None, "automation proposal record", ) } pub fn append_audit_event(paths: &AppPaths, event: &AuditEventRecord) -> Result<()> { paths.ensure()?; - append_jsonl_record(&paths.audit_events_path(), event, "audit event record") + append_jsonl_record( + &paths.audit_events_path(), + event, + Some(EVENT_LOG_ROTATION), + "audit event record", + ) } pub fn load_automation_proposals(paths: &AppPaths) -> Result> { @@ -761,19 +1041,39 @@ pub fn load_automation_proposals(paths: &AppPaths) -> Result Result> { - if limit == 0 { - return Ok(Vec::new()); + let mut proposals: Vec = load_recent_jsonl_records( + &paths.automation_proposals_path(), + 0, + limit, + "proposal record", + )?; + if proposals + .iter() + .any(|proposal| proposal.proposal_id.is_empty()) + { + proposals = load_automation_proposals(paths)?; + let skip = proposals.len().saturating_sub(limit); + proposals.drain(..skip); } - let mut proposals = load_automation_proposals(paths)?; proposals.reverse(); - proposals.truncate(limit); Ok(proposals) } +/// Find a proposal by id. Reads the whole store: a pending proposal can be +/// arbitrarily old. Compaction keeps that bounded by the pending proposals +/// plus [`REVIEWED_PROPOSALS_RETAINED`]. pub fn find_automation_proposal( paths: &AppPaths, proposal_id: &str, @@ -784,11 +1084,17 @@ pub fn find_automation_proposal( .with_context(|| format!("automation proposal `{proposal_id}` not found")) } -/// Set one proposal's `status` by rewriting the proposals store. +/// Set one proposal's `status` by rewriting the proposals store, compacting +/// it on the way (see [`compact_reviewed_proposals`]). /// /// The whole read-modify-write runs under the store's [`jsonl_lock_path`] /// lock, so a proposal appended (or another status set) meanwhile is neither /// dropped nor overwritten. +/// +/// The rewrite also migrates legacy proposals, written before proposals had +/// ids: each is written back with the id it was read under, derived from its +/// line number *before* compaction removed any line above it, so its id never +/// changes — and once written, reads no longer need the whole file to derive it. pub fn update_automation_proposal_status( paths: &AppPaths, proposal_id: &str, @@ -810,12 +1116,64 @@ pub fn update_automation_proposal_status( proposal.reviewed_at_unix_ms = Some(unix_time_millis()); } let updated = proposal.clone(); + compact_reviewed_proposals(&mut proposals, REVIEWED_PROPOSALS_RETAINED, proposal_id); write_automation_proposals(paths, &proposals)?; Ok(updated) } +/// Drop all but the `keep` most recently reviewed proposals, keeping every +/// pending one and the on-disk order of the rest. +/// +/// "Most recently reviewed" goes by `reviewed_at_unix_ms`, except that +/// `just_reviewed`, the proposal the calling status update reviewed, always +/// counts as the newest — even if it was queued long ago or the clock has +/// moved backwards since other reviews — so a review never removes the +/// proposal it just recorded. A reviewed proposal with no review time (older +/// records) counts as the oldest; ties go by position in the store. +fn compact_reviewed_proposals( + proposals: &mut Vec, + keep: usize, + just_reviewed: &str, +) { + let mut reviewed: Vec<(bool, u128, usize)> = proposals + .iter() + .enumerate() + .filter(|(_, proposal)| proposal.status != "pending") + .map(|(index, proposal)| { + ( + proposal.proposal_id == just_reviewed, + proposal.reviewed_at_unix_ms.unwrap_or(0), + index, + ) + }) + .collect(); + let Some(excess) = reviewed + .len() + .checked_sub(keep) + .filter(|&excess| excess > 0) + else { + return; + }; + reviewed.sort_unstable(); + let dropped: std::collections::HashSet = reviewed[..excess] + .iter() + .map(|&(_, _, index)| index) + .collect(); + let mut index = 0; + proposals.retain(|_| { + let kept = !dropped.contains(&index); + index += 1; + kept + }); +} + pub fn load_recent_audit_events(paths: &AppPaths, limit: usize) -> Result> { - load_recent_jsonl_records(&paths.audit_events_path(), limit, "audit event") + load_recent_jsonl_records( + &paths.audit_events_path(), + EVENT_LOG_ROTATED_GENERATIONS, + limit, + "audit event", + ) } /// Replace the proposals store with `proposals`. The caller must hold the @@ -871,7 +1229,12 @@ pub fn load_recent_automation_events( paths: &AppPaths, limit: usize, ) -> Result> { - load_recent_jsonl_records(&paths.automation_events_path(), limit, "event") + load_recent_jsonl_records( + &paths.automation_events_path(), + EVENT_LOG_ROTATED_GENERATIONS, + limit, + "event", + ) } #[cfg(test)] @@ -879,6 +1242,25 @@ mod tests { use super::*; use crate::test_support::temp_app_paths; + #[test] + fn a_rotation_failure_is_reported_once_per_store() { + // Shaped like the real stores, which share a file name and differ + // only by directory: keying on the file name alone must not pass. + let events = Path::new("rotation-warning-test/automations/events.jsonl"); + let audit = Path::new("rotation-warning-test/audit/events.jsonl"); + let error = anyhow::anyhow!("rename blocked"); + + let first = rotation_failure_warning(events, &error) + .expect("the first failure for a store is reported"); + assert!(first.contains("rotation-warning-test/automations/events.jsonl")); + assert!(first.contains("rename blocked")); + assert_eq!(rotation_failure_warning(events, &error), None); + let other = rotation_failure_warning(audit, &error) + .expect("one store's failure must not silence another store's warning"); + assert!(other.contains("rotation-warning-test/audit/events.jsonl")); + assert_eq!(rotation_failure_warning(audit, &error), None); + } + #[test] fn builtin_watchers_include_read_only_gpu_metrics() { let watcher = builtin_watcher("gpu-metrics").expect("gpu-metrics watcher should exist"); @@ -2208,4 +2590,702 @@ mod tests { paths.automations_dir().join("proposals.jsonl.lock") ); } + + #[test] + fn rotated_generations_are_numbered_siblings_of_the_store() { + let path = PathBuf::from("d").join("audit").join("events.jsonl"); + assert_eq!(jsonl_generation_path(&path, 0), path); + assert_eq!( + jsonl_generation_path(&path, 1), + PathBuf::from("d").join("audit").join("events.jsonl.1") + ); + assert_eq!( + jsonl_generation_path(&path, 4), + PathBuf::from("d").join("audit").join("events.jsonl.4") + ); + } + + /// Append an audit record tagged `tag` to `path` under `rotation`. + fn append_rotating(path: &Path, tag: &str, rotation: JsonlRotation) -> Result<()> { + append_jsonl_record( + path, + &audit_record(tag), + Some(rotation), + "audit event record", + ) + } + + /// The tags in every generation of the audit-shaped store at `path`, + /// oldest first, read straight from the files. + fn tags_in_every_generation(path: &Path, generations: usize) -> Result> { + let mut tags = Vec::new(); + for generation in (0..=generations).rev() { + let generation_path = jsonl_generation_path(path, generation); + if !generation_path.exists() { + continue; + } + for line in fs::read_to_string(&generation_path)?.lines() { + tags.push(serde_json::from_str::(line)?.message); + } + } + Ok(tags) + } + + fn recent_tags(path: &Path, generations: usize, limit: usize) -> Result> { + Ok( + load_recent_jsonl_records::(path, generations, limit, "audit")? + .into_iter() + .map(|record| record.message) + .collect(), + ) + } + + #[test] + fn rotation_keeps_the_live_file_and_the_configured_generations() -> Result<()> { + let (root, paths) = temp_app_paths("rotation-generations"); + paths.ensure()?; + let path = paths.audit_events_path(); + // One record per file: every append after the first rotates. + let rotation = JsonlRotation { + max_bytes: 1, + generations: 2, + }; + for tag in ["a", "b", "c", "d"] { + append_rotating(&path, tag, rotation)?; + } + let on_disk = tags_in_every_generation(&path, 3); + let third = jsonl_generation_path(&path, 3).exists(); + let lock = jsonl_lock_path(&path).is_file(); + let all = recent_tags(&path, 2, 10); + let two = recent_tags(&path, 2, 2); + fs::remove_dir_all(root).ok(); + // `a` was dropped by the rotation that would have made it `.3`. + assert_eq!(on_disk?, ["b", "c", "d"]); + assert!(!third, "only `generations` rotated files are kept"); + assert!(lock, "rotation leaves the store's lock file alone"); + assert_eq!(all?, ["b", "c", "d"]); + assert_eq!(two?, ["c", "d"]); + Ok(()) + } + + fn event_line(tag: &str) -> String { + serde_json::to_string(&event_record(tag)).expect("serialize event record") + } + + /// Write `contents` as the automation events store, creating its directory. + fn seed_events_store(paths: &AppPaths, contents: &str) -> Result<()> { + paths.ensure()?; + fs::write(paths.automation_events_path(), contents)?; + Ok(()) + } + + // The rotation tests above all drive `append_jsonl_record` and + // `load_recent_jsonl_records` directly with a custom `JsonlRotation`, + // which proves the mechanism works but never exercises the wiring that + // connects it to the real stores: `append_automation_event` and + // `append_audit_event` passing `Some(EVENT_LOG_ROTATION)`, and + // `load_recent_automation_events`/`load_recent_audit_events` passing + // `EVENT_LOG_ROTATED_GENERATIONS`. Flip either side back to `None`/`0` + // and every test above still passes. These two tests close that gap by + // going through the public wrappers only, at the real + // `EVENT_LOG_ROTATE_BYTES` cap — seeding the live file directly (fast) + // rather than looping a real append to size, since it's the one real + // append afterward that is under test, not how the filler got there. + #[test] + fn events_store_rotates_through_the_public_wrapper() -> Result<()> { + let (root, paths) = temp_app_paths("events-public-wrapper-rotation"); + let filler = event_line("filler") + "\n"; + let repeats = (EVENT_LOG_ROTATE_BYTES as usize).div_ceil(filler.len()); + seed_events_store(&paths, &filler.repeat(repeats))?; + + append_automation_event(&paths, &event_record("new"))?; + let rotated = jsonl_generation_path(&paths.automation_events_path(), 1).exists(); + // `limit: 2` so the read must actually cross into the rotated `.1` to + // satisfy it — `limit: 1` would be answered by the live file alone and + // never exercise `load_recent_automation_events`'s own + // `EVENT_LOG_ROTATED_GENERATIONS` wiring, only the append side's. + let recent = load_recent_automation_events(&paths, 2); + fs::remove_dir_all(root).ok(); + + assert!( + rotated, + "the live file was at the cap and should have rotated" + ); + assert_eq!( + recent? + .into_iter() + .map(|event| event.message) + .collect::>(), + ["filler", "new"] + ); + Ok(()) + } + + #[test] + fn audit_log_rotates_through_the_public_wrapper() -> Result<()> { + let (root, paths) = temp_app_paths("audit-public-wrapper-rotation"); + let filler = audit_line("filler") + "\n"; + let repeats = (EVENT_LOG_ROTATE_BYTES as usize).div_ceil(filler.len()); + seed_audit_store(&paths, &filler.repeat(repeats))?; + + append_audit_event(&paths, &audit_record("new"))?; + let rotated = jsonl_generation_path(&paths.audit_events_path(), 1).exists(); + // `limit: 2`, for the same reason as the events-store twin above: the + // read must actually cross into `.1` to exercise + // `load_recent_audit_events`'s own generations wiring. + let recent = load_recent_audit_events(&paths, 2); + fs::remove_dir_all(root).ok(); + + assert!( + rotated, + "the live file was at the cap and should have rotated" + ); + assert_eq!( + recent? + .into_iter() + .map(|event| event.message) + .collect::>(), + ["filler", "new"] + ); + Ok(()) + } + + // Right after a rollover the live file holds fewer records than a reader + // asks for; the rest must come from the rotated generations, in order. + #[test] + fn load_recent_spans_a_rollover() -> Result<()> { + let (root, paths) = temp_app_paths("rotation-spans"); + paths.ensure()?; + let path = paths.audit_events_path(); + let line_len = audit_line("00").len() as u64 + 1; + // Rotate once a file holds five records. + let rotation = JsonlRotation { + max_bytes: 5 * line_len, + generations: 4, + }; + for n in 0..12 { + append_rotating(&path, &format!("{n:02}"), rotation)?; + } + let live = fs::read_to_string(&path)?.lines().count(); + let recent = recent_tags(&path, rotation.generations, 4); + let spanning = recent_tags(&path, rotation.generations, 9); + let everything = recent_tags(&path, rotation.generations, 100); + let live_only = recent_tags(&path, 0, 100); + fs::remove_dir_all(root).ok(); + assert_eq!(live, 2, "12 records rotate after the 5th and 10th"); + assert_eq!(recent?, ["08", "09", "10", "11"]); + assert_eq!( + spanning?, + ["03", "04", "05", "06", "07", "08", "09", "10", "11"] + ); + let expected: Vec = (0..12).map(|n| format!("{n:02}")).collect(); + assert_eq!(everything?, expected); + assert_eq!(live_only?, ["10", "11"]); + Ok(()) + } + + // A reader can see the store mid-rotation: no live file yet, or a gap + // where `.1` was shifted to `.2` but the live file not yet renamed. + #[test] + fn load_recent_reads_a_store_caught_mid_rotation_in_order() -> Result<()> { + let (root, paths) = temp_app_paths("rotation-mid"); + paths.ensure()?; + let path = paths.audit_events_path(); + let write = |generation: usize, tags: &[&str]| -> Result<()> { + let text: String = tags.iter().map(|tag| audit_line(tag) + "\n").collect(); + fs::write(jsonl_generation_path(&path, generation), text)?; + Ok(()) + }; + write(1, &["c", "d"])?; + write(2, &["a", "b"])?; + let no_live = recent_tags(&path, 4, 3); + write(0, &["e"])?; + fs::remove_file(jsonl_generation_path(&path, 1))?; + write(3, &["a"])?; + write(2, &["b", "c", "d"])?; + let gap = recent_tags(&path, 4, 10); + fs::remove_dir_all(root).ok(); + assert_eq!(no_live?, ["b", "c", "d"]); + assert_eq!(gap?, ["a", "b", "c", "d", "e"]); + Ok(()) + } + + // Rotation is best effort: a rename that fails must not cost the record, + // nor rename the live file over a `.1` that was never shifted away. + #[test] + fn a_rotation_that_fails_still_appends_the_record() -> Result<()> { + let (root, paths) = temp_app_paths("rotation-fails"); + paths.ensure()?; + let path = paths.audit_events_path(); + fs::write(&path, audit_line("a") + "\n")?; + fs::write(jsonl_generation_path(&path, 1), audit_line("old") + "\n")?; + // `.1` cannot be renamed over a non-empty directory. + let blocker = jsonl_generation_path(&path, 2); + fs::create_dir_all(&blocker)?; + fs::write(blocker.join("keep"), "x")?; + let rotation = JsonlRotation { + max_bytes: 1, + generations: 2, + }; + let appended = append_rotating(&path, "b", rotation); + let live = fs::read_to_string(&path); + let first = fs::read_to_string(jsonl_generation_path(&path, 1)); + let recent = recent_tags(&path, 2, 10); + fs::remove_dir_all(root).ok(); + appended?; + assert_eq!(live?, format!("{}\n{}\n", audit_line("a"), audit_line("b"))); + assert_eq!(first?, audit_line("old") + "\n"); + assert_eq!(recent?, ["old", "a", "b"]); + Ok(()) + } + + // `rotation_failure_warning` is unit-tested on its own; this pins that + // `append_jsonl_record` actually prints it, once for a store that keeps + // failing to rotate. Run in a child process to read its stderr, which the + // test harness would otherwise swallow. + #[test] + fn a_failed_rotation_prints_its_warning_once_to_stderr() -> Result<()> { + const WARNING_CHILD_ENV: &str = "ROCM_CORE_TEST_ROTATION_WARNING_CHILD"; + + if let Some(path) = std::env::var_os(WARNING_CHILD_ENV) { + let rotation = JsonlRotation { + max_bytes: 1, + generations: 2, + }; + append_rotating(Path::new(&path), "b", rotation)?; + append_rotating(Path::new(&path), "c", rotation)?; + return Ok(()); + } + + let (root, paths) = temp_app_paths("rotation-warning"); + paths.ensure()?; + let path = paths.audit_events_path(); + fs::write(&path, audit_line("a") + "\n")?; + fs::write(jsonl_generation_path(&path, 1), audit_line("old") + "\n")?; + // `.1` cannot be renamed over a non-empty directory. + let blocker = jsonl_generation_path(&path, 2); + fs::create_dir_all(&blocker)?; + fs::write(blocker.join("keep"), "x")?; + let exe = std::env::current_exe().context("current test executable")?; + let output = std::process::Command::new(&exe) + .args([ + "automation::tests::a_failed_rotation_prints_its_warning_once_to_stderr", + "--exact", + "--quiet", + "--nocapture", + ]) + .env(WARNING_CHILD_ENV, &path) + .output() + .context("spawn rotation-warning child")?; + fs::remove_dir_all(root).ok(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + output.status.success(), + "child process failed: {:?}\nstdout: {}\nstderr: {stderr}", + output.status, + String::from_utf8_lossy(&output.stdout), + ); + assert_eq!( + stderr.matches("warning: failed to rotate").count(), + 1, + "two appends to a store stuck at its cap must print one warning: {stderr}" + ); + assert!( + stderr.contains("events.jsonl"), + "the warning names the store: {stderr}" + ); + Ok(()) + } + + // A rotation that shifted `.1` away but then failed to rename the live + // file leaves a gap at `.1`. The next rotation must only fill it: shifting + // the older generations again would drop one more each time a rotation + // keeps failing that way (e.g. on Windows, while another program holds the + // live file open without delete sharing). + #[test] + fn a_rotation_after_one_left_part_way_drops_no_generation() -> Result<()> { + let (root, paths) = temp_app_paths("rotation-part-way"); + paths.ensure()?; + let path = paths.audit_events_path(); + let write = |generation: usize, tag: &str| -> Result<()> { + fs::write( + jsonl_generation_path(&path, generation), + audit_line(tag) + "\n", + )?; + Ok(()) + }; + // `.1` = a was shifted to `.2`; the live file `e` was not renamed. + write(0, "e")?; + write(2, "a")?; + write(3, "b")?; + write(4, "c")?; + let rotation = JsonlRotation { + max_bytes: 1, + generations: 4, + }; + append_rotating(&path, "f", rotation)?; + let on_disk = tags_in_every_generation(&path, 5); + let fifth = jsonl_generation_path(&path, 5).exists(); + fs::remove_dir_all(root).ok(); + assert_eq!(on_disk?, ["c", "b", "a", "e", "f"]); + assert!(!fifth, "rotation never writes past `generations`"); + Ok(()) + } + + fn assert_holds_exactly_in_writer_order(tags: &[String], expected: &[String]) { + let mut sorted = tags.to_vec(); + sorted.sort(); + let mut expected = expected.to_vec(); + expected.sort(); + assert_eq!( + sorted, expected, + "rotation lost or duplicated records under concurrent writers" + ); + // Each writer appends in order, so its records must read back in order + // across the generations, oldest first. + let mut last_seen = std::collections::HashMap::new(); + for tag in tags { + let (writer, n) = tag.rsplit_once('-').expect("tag is -"); + let n: usize = n.parse().expect("numeric sequence"); + if let Some(previous) = last_seen.insert(writer.to_owned(), n) { + assert!( + previous < n, + "{writer}'s records are out of order: {tags:?}" + ); + } + } + } + + /// Small enough that the concurrency tests below rotate every ~14 + /// records, with enough generations that none is dropped. + const TEST_ROTATION: JsonlRotation = JsonlRotation { + max_bytes: 2048, + generations: 64, + }; + + // Rotation renames the live file, so it must run under the store lock: + // an append that had already opened the file being renamed would land in + // the rotated copy (or be lost), and two writers rotating at once would + // shift the generations twice. + #[test] + fn rotation_under_concurrent_threads_loses_no_records() -> Result<()> { + const WRITERS: usize = 8; + const APPENDS_PER_WRITER: usize = 25; + let (root, paths) = temp_app_paths("rotation-threads"); + paths.ensure()?; + let path = paths.audit_events_path(); + let barrier = std::sync::Barrier::new(WRITERS); + std::thread::scope(|scope| { + let handles: Vec<_> = (0..WRITERS) + .map(|writer| { + let (path, barrier) = (&path, &barrier); + scope.spawn(move || -> Result<()> { + barrier.wait(); + for n in 0..APPENDS_PER_WRITER { + append_rotating(path, &format!("t{writer}-{n}"), TEST_ROTATION)?; + } + Ok(()) + }) + }) + .collect(); + for handle in handles { + handle.join().expect("writer thread panicked")?; + } + Ok::<_, anyhow::Error>(()) + })?; + + let expected: Vec = (0..WRITERS) + .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("t{writer}-{n}"))) + .collect(); + let tags = tags_in_every_generation(&path, TEST_ROTATION.generations); + let rotated = jsonl_generation_path(&path, 2).exists(); + fs::remove_dir_all(root).ok(); + assert!(rotated, "the test must actually rotate more than once"); + assert_holds_exactly_in_writer_order(&tags?, &expected); + Ok(()) + } + + /// Env var that turns a re-exec of this test binary into one writer of + /// [`rotation_under_concurrent_processes_loses_no_records`]: its value is + /// `|`. + const CHILD_ROTATING_WRITER_ENV: &str = "ROCM_CORE_TEST_JSONL_CHILD_ROTATING_WRITER"; + + #[test] + fn rotation_under_concurrent_processes_loses_no_records() -> Result<()> { + const WRITERS: usize = 6; + const APPENDS_PER_WRITER: usize = 20; + if let Some(spec) = std::env::var_os(CHILD_ROTATING_WRITER_ENV) { + let spec = spec.to_string_lossy().into_owned(); + let (path, writer) = spec.split_once('|').context("malformed writer spec")?; + for n in 0..APPENDS_PER_WRITER { + append_rotating(Path::new(path), &format!("p{writer}-{n}"), TEST_ROTATION)?; + } + return Ok(()); + } + + let (root, paths) = temp_app_paths("rotation-processes"); + paths.ensure()?; + let path = paths.audit_events_path(); + let exe = std::env::current_exe().context("current test executable")?; + let children = (0..WRITERS) + .map(|writer| { + std::process::Command::new(&exe) + .args([ + "automation::tests::rotation_under_concurrent_processes_loses_no_records", + "--exact", + "--quiet", + ]) + .env( + CHILD_ROTATING_WRITER_ENV, + format!("{}|{writer}", path.display()), + ) + .stdout(std::process::Stdio::null()) + .spawn() + .context("spawn writer process") + }) + .collect::>>()?; + for mut child in children { + let status = child.wait()?; + assert!(status.success(), "writer process failed: {status:?}"); + } + + let expected: Vec = (0..WRITERS) + .flat_map(|writer| (0..APPENDS_PER_WRITER).map(move |n| format!("p{writer}-{n}"))) + .collect(); + let tags = tags_in_every_generation(&path, TEST_ROTATION.generations); + let rotated = jsonl_generation_path(&path, 2).exists(); + fs::remove_dir_all(root).ok(); + assert!(rotated, "the test must actually rotate more than once"); + assert_holds_exactly_in_writer_order(&tags?, &expected); + Ok(()) + } + + // Readers take no lock, so a rotation can rename the files they are + // opening. One sequential writer makes every correct read a run of + // consecutive numbers: a file read twice or a generation skipped breaks + // the run. + #[test] + fn reads_racing_rotations_see_each_record_once_and_in_order() -> Result<()> { + const APPENDS: usize = 4000; + const LIMIT: usize = 20; + let rotation = JsonlRotation { + max_bytes: 2048, + generations: 3, + }; + let (root, paths) = temp_app_paths("rotation-read-race"); + paths.ensure()?; + let path = paths.audit_events_path(); + let done = std::sync::atomic::AtomicBool::new(false); + let outcome = std::thread::scope(|scope| { + let reader = scope.spawn(|| -> Result { + let mut reads = 0; + while !done.load(std::sync::atomic::Ordering::Acquire) { + let numbers: Vec = recent_tags(&path, rotation.generations, LIMIT)? + .iter() + .map(|tag| tag.parse().expect("numeric tag")) + .collect(); + if numbers.windows(2).any(|pair| pair[1] != pair[0] + 1) { + bail!("read {numbers:?}: a record was read twice or skipped"); + } + // Three full generations always hold more than LIMIT. + if numbers.last().is_some_and(|&last| last >= 100) && numbers.len() != LIMIT { + bail!("read {numbers:?}: a generation was skipped"); + } + reads += 1; + } + Ok(reads) + }); + let written = + (0..APPENDS).try_for_each(|n| append_rotating(&path, &n.to_string(), rotation)); + done.store(true, std::sync::atomic::Ordering::Release); + let reads = reader.join().expect("reader panicked"); + written.and(reads) + }); + fs::remove_dir_all(root).ok(); + assert!(outcome? > 0, "the reader must have run"); + Ok(()) + } + + fn reviewed_proposal(tag: &str, reviewed_at: Option) -> AutomationProposalRecord { + AutomationProposalRecord { + status: "approved".to_owned(), + reviewed_at_unix_ms: reviewed_at, + ..proposal_record(tag) + } + } + + fn proposal_ids(proposals: &[AutomationProposalRecord]) -> Vec<&str> { + proposals.iter().map(|p| p.proposal_id.as_str()).collect() + } + + #[test] + fn compaction_keeps_every_pending_proposal_and_the_most_recently_reviewed() { + let mut proposals = vec![ + proposal_record("pending-old"), + reviewed_proposal("reviewed-at-10", Some(10)), + reviewed_proposal("reviewed-at-30", Some(30)), + proposal_record("pending-new"), + reviewed_proposal("reviewed-at-20", Some(20)), + reviewed_proposal("reviewed-never-timed", None), + ]; + compact_reviewed_proposals(&mut proposals, 2, "pending-new"); + assert_eq!( + proposal_ids(&proposals), + [ + "pending-old", + "reviewed-at-30", + "pending-new", + "reviewed-at-20" + ] + ); + compact_reviewed_proposals(&mut proposals, 2, "pending-new"); + assert_eq!( + proposals.len(), + 4, + "already within the cap: nothing dropped" + ); + // The proposal just reviewed is kept even when its review time is the + // oldest, as after the clock moved backwards. + compact_reviewed_proposals(&mut proposals, 1, "reviewed-at-20"); + assert_eq!( + proposal_ids(&proposals), + ["pending-old", "pending-new", "reviewed-at-20"] + ); + } + + /// Write `proposals` as the proposals store, one JSON line each. + fn seed_proposals_store(paths: &AppPaths, lines: &[String]) -> Result<()> { + paths.ensure()?; + fs::write(paths.automation_proposals_path(), lines.join("\n") + "\n")?; + Ok(()) + } + + fn proposal_line(proposal: &AutomationProposalRecord) -> String { + serde_json::to_string(proposal).expect("serialize proposal") + } + + // A pending proposal must stay findable by id until it is reviewed, however + // many proposals were reviewed after it was queued. + #[test] + fn a_status_update_compacts_reviewed_proposals_but_keeps_pending_ones() -> Result<()> { + let (root, paths) = temp_app_paths("proposal-compaction"); + let reviewed = REVIEWED_PROPOSALS_RETAINED + 5; + let mut lines = vec![proposal_line(&proposal_record("pending-old"))]; + lines.extend((0..reviewed).map(|n| { + proposal_line(&reviewed_proposal( + &format!("r{n}"), + Some(1_000 + n as u128), + )) + })); + lines.push(proposal_line(&proposal_record("to-review"))); + seed_proposals_store(&paths, &lines)?; + + let updated = update_automation_proposal_status(&paths, "to-review", "rejected")?; + let proposals = load_automation_proposals(&paths); + let pending_old = find_automation_proposal(&paths, "pending-old"); + fs::remove_dir_all(root).ok(); + + let proposals = proposals?; + assert_eq!(updated.status, "rejected"); + assert_eq!(pending_old?.status, "pending"); + let reviewed_ids: Vec<&str> = proposals + .iter() + .filter(|p| p.status != "pending") + .map(|p| p.proposal_id.as_str()) + .collect(); + assert_eq!(reviewed_ids.len(), REVIEWED_PROPOSALS_RETAINED); + // The just-reviewed proposal is the newest review, so it is kept; the + // six oldest reviews make room for it. + assert_eq!(reviewed_ids.last(), Some(&"to-review")); + for n in 0..6 { + assert!( + !reviewed_ids.contains(&format!("r{n}").as_str()), + "r{n} kept" + ); + } + assert_eq!(reviewed_ids[0], "r6"); + assert_eq!(proposals[0].proposal_id, "pending-old"); + Ok(()) + } + + // A legacy proposal's id derives from its line number, so compaction + // removing lines above it must not change the id it is known by — and + // the rewrite must persist that id so later reads need no line number. + #[test] + fn legacy_proposal_ids_survive_compaction_and_are_persisted() -> Result<()> { + let (root, paths) = temp_app_paths("proposal-legacy-compaction"); + let reviewed = REVIEWED_PROPOSALS_RETAINED + 3; + let mut lines: Vec = (0..reviewed) + .map(|n| proposal_line(&reviewed_proposal(&format!("r{n}"), Some(n as u128 + 1)))) + .collect(); + let legacy = AutomationProposalRecord { + at_unix_ms: 77, + ..proposal_record("") + }; + lines.push(proposal_line(&legacy)); + lines.push(proposal_line(&proposal_record("to-review"))); + seed_proposals_store(&paths, &lines)?; + let legacy_id = format!("legacy-77-{reviewed}"); + + let before = find_automation_proposal(&paths, &legacy_id); + let update = update_automation_proposal_status(&paths, "to-review", "approved"); + let after = find_automation_proposal(&paths, &legacy_id); + let recent = load_recent_automation_proposals(&paths, 2); + let on_disk = fs::read_to_string(paths.automation_proposals_path()); + let approve_legacy = update_automation_proposal_status(&paths, &legacy_id, "approved"); + fs::remove_dir_all(root).ok(); + + assert_eq!(before?.status, "pending"); + update?; + assert_eq!(after?.status, "pending", "the legacy id still resolves"); + assert_eq!(proposal_ids(&recent?), ["to-review", legacy_id.as_str()]); + assert!( + on_disk?.contains(&format!("\"proposal_id\":\"{legacy_id}\"")), + "the rewrite persists the legacy id" + ); + assert_eq!(approve_legacy?.status, "approved"); + Ok(()) + } + + #[test] + fn recent_proposals_read_only_the_tail_but_fall_back_for_legacy_ids() -> Result<()> { + let (root, paths) = temp_app_paths("proposal-tail"); + // A corrupt line older than the window fails a whole-file read, so a + // read that succeeds proves only the tail was read. + seed_proposals_store( + &paths, + &[ + "not json".to_owned(), + proposal_line(&proposal_record("a")), + proposal_line(&proposal_record("b")), + ], + )?; + let tail = load_recent_automation_proposals(&paths, 2); + let whole = load_automation_proposals(&paths); + let legacy = AutomationProposalRecord { + at_unix_ms: 5, + ..proposal_record("") + }; + seed_proposals_store( + &paths, + &[ + proposal_line(&proposal_record("a")), + String::new(), + proposal_line(&legacy), + proposal_line(&proposal_record("b")), + ], + )?; + let with_legacy = load_recent_automation_proposals(&paths, 2); + let full = load_automation_proposals(&paths); + fs::remove_dir_all(root).ok(); + assert_eq!(proposal_ids(&tail?), ["b", "a"]); + assert!( + whole.is_err(), + "the whole-file read reports the corrupt line" + ); + // Same ids as the whole-file read, line index and all. + assert_eq!(proposal_ids(&with_legacy?), ["b", "legacy-5-2"]); + assert_eq!(proposal_ids(&full?), ["a", "legacy-5-2", "b"]); + Ok(()) + } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 9ea27e6074..82afa7e340 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -54,9 +54,10 @@ pub mod uv; pub mod windows_driver; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, - AutomationTriggerEvent, BuiltinWatcherSpec, WatcherRuntimeSnapshot, append_audit_event, - append_automation_event, append_automation_proposal, builtin_watcher, builtin_watchers, - find_automation_proposal, generate_proposal_id, load_automation_proposals, + AutomationTriggerEvent, BuiltinWatcherSpec, EVENT_LOG_ROTATE_BYTES, + EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, WatcherRuntimeSnapshot, + append_audit_event, append_automation_event, append_automation_proposal, builtin_watcher, + builtin_watchers, find_automation_proposal, generate_proposal_id, load_automation_proposals, load_recent_audit_events, load_recent_automation_events, load_recent_automation_proposals, update_automation_proposal_status, }; diff --git a/docs/automations.md b/docs/automations.md index 1b341d16f5..e77ddec78b 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -86,6 +86,17 @@ the exact `/automations approve ` and `/automations reject ` controls. Raw backend action and restricted-tool names are kept out of this overview so the user-facing history stays readable. +In `propose` mode, while a restart request for a managed server is still +awaiting review, `server-recover` does not queue another one for that server. +It records a +`restart_proposal_already_pending` event instead, so a server that stays +unhealthy shows one pending request rather than a new one on every recovery +check. If the review requests cannot be read (for example the file is +corrupt), `server-recover` records a `restart_proposal_check_failed` error +event and queues no restart for that server until they can be read again: it +cannot tell whether one is already pending, and the failure is reported rather +than raised so the `rocmd` daemon keeps running. + For `cache-warm`, the webhook accepts exactly `cache.warm` and can only request a reviewed prefetch request for an artifact ref already present in the model recipe registry: @@ -164,6 +175,33 @@ Invoke-RestMethod ` http://127.0.0.1:18080/automation-events ``` +## History And Retention + +Automations keep three local histories under the rocm-cli data directory +(`~/.rocm` by default): + +- `automations/events.jsonl`: what each watcher did. +- `audit/events.jsonl`: the audit log, written by both `rocm` and `rocmd`. +- `automations/proposals.jsonl`: review requests and their status. + +The event log and the audit log are capped by size. When one reaches 8 MiB, +the next write renames it to `events.jsonl.1` (shifting an existing `.1` to +`.2`, and so on) and starts a new file. Four rotated files are kept, so each +log holds at most about 40 MiB; the oldest is deleted by the next rotation. +With the `gpu-metrics` watcher enabled, the busiest writer, that is roughly +two months of history; without it, considerably more. `rocm automations` and +the dashboard read across the rotated files, so recent activity stays visible +right after a rotation. + +A log that already grew past 8 MiB before this cap existed is rotated aside +whole, so it can take up to four more rotations before that oversized file is +deleted and the log falls under the cap. + +Review requests are never rotated away, because a pending request must stay +available to approve or reject. Instead, approving or rejecting a request +trims the history: every pending request is kept, along with the 200 most +recently reviewed ones, and older reviewed requests are removed. + ## Artifact Prefetch Action The reviewed artifact-prefetch action keeps downloads disabled unless the caller diff --git a/docs/testing.md b/docs/testing.md index 1ff01a513c..e7b6037618 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1414,6 +1414,30 @@ The plain proposal history test verifies that `/automations` and server/artifact details, and approve/reject controls instead of raw backend action or restricted-tool names. +Automation history stores (events, audit, proposals): + +```bash +cargo test -p rocm-core automation::tests +``` + +These cover the three JSONL stores in +[automation.rs](../crates/rocm-core/src/automation.rs): concurrent writers +across threads and processes losing no records, size-based rotation of the +event and audit logs under the same lock (no record lost or duplicated while +writers rotate, recent reads spanning the rollover, a failed rotation still +keeping the record and printing its one-shot warning to stderr, lock-free +readers never reading a file twice or skipping a generation), and proposal +compaction keeping every pending proposal, the +newest reviewed ones, and legacy proposal ids. Most of those drive +`append_jsonl_record`/`load_recent_jsonl_records` directly with a custom +rotation policy; two further tests go through the public +`append_automation_event`/`append_audit_event` and +`load_recent_automation_events`/`load_recent_audit_events` wrappers instead, +at the real size cap, so the wiring between them and the rotation policy +itself stays covered. The user-visible side is `automations-07` through +`automations-09` in +[automations.feature](../tests/e2e-cucumber/features/automations.feature). + Natural-language plan handoff tests: ```bash diff --git a/tests/e2e-cucumber/features/automations.feature b/tests/e2e-cucumber/features/automations.feature index a195588c0b..372846979c 100644 --- a/tests/e2e-cucumber/features/automations.feature +++ b/tests/e2e-cucumber/features/automations.feature @@ -5,8 +5,9 @@ Feature: Automation watchers # automations` lists the most recent review requests (any status — only the # per-entry approve/reject controls are gated on "pending"). This feature # pins the enable/disable/mode-confirmation slice, the proposal-history - # listing's unit formatting, and the bare listing's tolerance of a torn - # audit-log tail; the watcher/daemon trigger behaviour itself is covered + # listing's unit formatting, the bare listing's tolerance of a torn + # audit-log tail, and how the audit log and proposal history are kept + # bounded; the watcher/daemon trigger behaviour itself is covered # elsewhere. Config-only (no GPU, no network), so it runs on the mock lane # every PR. @@ -83,3 +84,46 @@ Feature: Automation watchers And the audit event log ends in a torn, unterminated line When the user lists automations Then the CLI still lists the record before the torn tail + + # The automation event log and the audit log are size-capped (ROCMAI-729): + # the write that finds the live log at its cap first renames it aside to + # `events.jsonl.1`, keeping a fixed number of such generations. This seeds the + # audit log at the cap, makes one change the CLI audits, and checks both the + # files and that the automations listing still shows activity from before the + # rollover, which it can only read from the rotated generation. + @id:automations-audit-log-rotates-at-cap + Scenario: automations-07 - A full audit log is set aside and its recent history stays visible + Given an audit log that has reached its size cap + When the user selects a permissions mode + Then the full audit log is kept aside as its first rotated generation + And the permissions change starts a fresh audit log + When the user lists automations + Then the recent background activity still includes entries from before the rotation + + # The proposal history is compacted, not rotated (ROCMAI-729): a pending + # proposal must stay findable by id until it is reviewed, so reviewing one + # rewrites the history keeping every pending proposal but only the most + # recently reviewed ones. Approves through `rocm mcp-call proposal_action`, the + # hidden entry point the dashboard and chat review controls use. + @id:automations-review-compaction-keeps-pending + Scenario: automations-08 - Approving a proposal trims old review history but keeps older pending proposals + Given a pending proposal queued before a long history of reviewed proposals + When the user approves the newest pending proposal + Then the CLI confirms the approval + And only the most recently reviewed proposals are kept + When the user looks up the older pending proposal + Then the CLI shows it still pending + + # automations-07 only exercises a single rollover: a live file at the cap + # gets a `.1`. The shift-and-drop-oldest chain beyond that — `.1` to `.2`, + # `.2` to `.3`, `.3` to `.4`, and the previous `.4` dropped — is otherwise + # pinned only by unit tests in `automation.rs`. This seeds a full chain of + # four already-rotated generations plus a live file at the cap, forces one + # more rotation, and checks the whole chain shifted down by one and the + # oldest generation's content is gone, not just that `.1` exists. + @id:automations-rotation-chain-shifts-and-drops-oldest + Scenario: automations-09 - A further rotation shifts the whole chain and drops the oldest generation + Given an audit log already holding a full chain of four rotated generations + When the user selects a permissions mode + Then each rotated generation holds what the previous one held + And the oldest previous generation is gone diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index cdc09c8905..3a3534860d 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -13,12 +13,17 @@ //! running under THIS test process's (live) pid, so the CLI's double-spawn //! guard skips the spawn. Contracts verified against the running Linux binary //! (EAI-8072). Scoped to the enable/disable/mode slice, the proposal-history -//! listing's unit formatting, and the bare listing's tolerance of a torn -//! audit-log tail; the watcher/daemon trigger behaviour itself is covered -//! separately. +//! listing's unit formatting, the bare listing's tolerance of a torn +//! audit-log tail, and the size cap on the audit log and compaction of the +//! proposal history (ROCMAI-729); the watcher/daemon trigger behaviour itself +//! is covered separately. use cucumber::{given, then, when}; -use rocm_core::{AuditEventRecord, AutomationProposalRecord, append_automation_proposal}; +use rocm_core::{ + AuditEventRecord, AutomationProposalRecord, EVENT_LOG_ROTATE_BYTES, + EVENT_LOG_ROTATED_GENERATIONS, REVIEWED_PROPOSALS_RETAINED, append_automation_proposal, + load_automation_proposals, +}; use crate::E2eWorld; @@ -185,6 +190,140 @@ async fn enabled_watcher(world: &mut E2eWorld) { ); } +/// Action of the audit records seeded by `audit_log_at_cap`. +const SEEDED_AUDIT_ACTION: &str = "e2e_seeded"; + +#[given("an audit log that has reached its size cap")] +async fn audit_log_at_cap(world: &mut E2eWorld) { + let paths = world.app_paths(); + std::fs::create_dir_all(paths.audit_dir()).expect("failed to create audit dir"); + let record = AuditEventRecord { + at_unix_ms: 1_700_000_000_000, + source: "rocm".to_owned(), + // `rocm automations` summarizes this category as "Install activity was + // recorded.", unlike the permissions change made afterwards. + category: "install".to_owned(), + actor: "cli".to_owned(), + level: "info".to_owned(), + action: SEEDED_AUDIT_ACTION.to_owned(), + message: "x".repeat(1024), + watcher_id: None, + service_id: None, + }; + let line = serde_json::to_string(&record).expect("serialize audit record") + "\n"; + let lines = EVENT_LOG_ROTATE_BYTES.div_ceil(line.len() as u64); + std::fs::write( + paths.audit_events_path(), + line.repeat(usize::try_from(lines).expect("line count fits usize")), + ) + .expect("failed to seed the audit log"); +} + +/// Action tag for the already-rotated `generation`-th file seeded by +/// `audit_log_with_full_rotation_chain`, so after one more rotation each +/// generation's new content can be traced back to which generation it came +/// from. +fn seeded_generation_action(generation: usize) -> String { + format!("e2e_seeded_gen{generation}") +} + +/// Action tag for the live file seeded by `audit_log_with_full_rotation_chain`, +/// distinct from every `seeded_generation_action` tag. +const SEEDED_LIVE_BEFORE_CHAIN_ROTATION: &str = "e2e_seeded_live_before_chain_rotation"; + +#[given("an audit log already holding a full chain of four rotated generations")] +async fn audit_log_with_full_rotation_chain(world: &mut E2eWorld) { + let paths = world.app_paths(); + std::fs::create_dir_all(paths.audit_dir()).expect("failed to create audit dir"); + let live = paths.audit_events_path(); + + // An already-rotated generation is never resized by a further rotation — + // only shifted or renamed over — so these only need to exist and be + // distinguishable, not reach the cap themselves. + for generation in 1..=EVENT_LOG_ROTATED_GENERATIONS { + let record = AuditEventRecord { + at_unix_ms: 1_700_000_000_000, + source: "rocm".to_owned(), + category: "install".to_owned(), + actor: "cli".to_owned(), + level: "info".to_owned(), + action: seeded_generation_action(generation), + message: format!("generation {generation}"), + watcher_id: None, + service_id: None, + }; + let line = serde_json::to_string(&record).expect("serialize audit record") + "\n"; + std::fs::write( + live.with_file_name(format!("events.jsonl.{generation}")), + line, + ) + .unwrap_or_else(|error| panic!("failed to seed generation {generation}: {error}")); + } + + // The live file itself must be at the cap, or the change made afterwards + // would not rotate anything at all. + let live_record = AuditEventRecord { + at_unix_ms: 1_700_000_000_000, + source: "rocm".to_owned(), + category: "install".to_owned(), + actor: "cli".to_owned(), + level: "info".to_owned(), + action: SEEDED_LIVE_BEFORE_CHAIN_ROTATION.to_owned(), + message: "x".repeat(1024), + watcher_id: None, + service_id: None, + }; + let live_line = serde_json::to_string(&live_record).expect("serialize audit record") + "\n"; + let lines = EVENT_LOG_ROTATE_BYTES.div_ceil(live_line.len() as u64); + std::fs::write( + &live, + live_line.repeat(usize::try_from(lines).expect("line count fits usize")), + ) + .expect("failed to seed the live audit log"); +} + +const OLD_PENDING_PROPOSAL: &str = "e2e-pending-old"; +const NEW_PENDING_PROPOSAL: &str = "e2e-pending-new"; +/// Reviewed proposals seeded beyond the retention cap, which compaction drops. +const EXCESS_REVIEWED: usize = 5; + +fn e2e_proposal(proposal_id: &str, reviewed_at: Option) -> AutomationProposalRecord { + AutomationProposalRecord { + at_unix_ms: 1_700_000_000_000, + proposal_id: proposal_id.to_owned(), + watcher_id: "therock-update".to_owned(), + action: "queue_update_proposal".to_owned(), + title: "check for ROCm updates".to_owned(), + message: "a newer ROCm runtime may be available".to_owned(), + status: if reviewed_at.is_some() { + "approved" + } else { + "pending" + } + .to_owned(), + service_id: None, + tool: Some("check_updates".to_owned()), + arguments: serde_json::json!({}), + reviewed_at_unix_ms: reviewed_at, + } +} + +#[given("a pending proposal queued before a long history of reviewed proposals")] +async fn pending_before_review_history(world: &mut E2eWorld) { + let paths = world.app_paths(); + let mut proposals = vec![e2e_proposal(OLD_PENDING_PROPOSAL, None)]; + proposals.extend((0..REVIEWED_PROPOSALS_RETAINED + EXCESS_REVIEWED).map(|n| { + e2e_proposal( + &format!("e2e-reviewed-{n}"), + Some(1_700_000_000_000 + n as u128), + ) + })); + proposals.push(e2e_proposal(NEW_PENDING_PROPOSAL, None)); + for proposal in &proposals { + append_automation_proposal(&paths, proposal).expect("failed to seed a proposal"); + } +} + // ── When ─────────────────────────────────────────────────────────── #[when("the user enables an automation watcher in observe mode")] @@ -233,6 +372,33 @@ async fn enable_unknown(world: &mut E2eWorld) { record(world, stdout, stderr, rc); } +/// Run the hidden `rocm mcp-call proposal_action` entry point the dashboard +/// and chat review controls use. +fn proposal_action(world: &mut E2eWorld, proposal_id: &str, action: &str) { + let arguments = serde_json::json!({ "proposal_id": proposal_id, "action": action }); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "mcp-call", + "proposal_action", + "--arguments-json", + &arguments.to_string(), + "--allow-mutation", + ], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user approves the newest pending proposal")] +async fn approve_newest_pending(world: &mut E2eWorld) { + proposal_action(world, NEW_PENDING_PROPOSAL, "approve"); +} + +#[when("the user looks up the older pending proposal")] +async fn show_old_pending(world: &mut E2eWorld) { + proposal_action(world, OLD_PENDING_PROPOSAL, "show"); +} + // ── Then ─────────────────────────────────────────────────────────── #[then("the CLI still lists the record before the torn tail")] @@ -324,6 +490,177 @@ async fn refuse_unknown(world: &mut E2eWorld) { ); } +/// The audit records in `path`, which must all parse. +fn audit_records(path: &std::path::Path) -> Vec { + std::fs::read_to_string(path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())) + .lines() + .enumerate() + .map(|(index, line)| { + serde_json::from_str(line).unwrap_or_else(|error| { + panic!( + "{}:{}: audit line failed to parse: {error}\n{line}", + path.display(), + index + 1 + ) + }) + }) + .collect() +} + +#[then("the full audit log is kept aside as its first rotated generation")] +async fn audit_log_rotated(world: &mut E2eWorld) { + ok_output(world); + let live = world.app_paths().audit_events_path(); + let rotated = live.with_file_name("events.jsonl.1"); + let size = std::fs::metadata(&rotated) + .unwrap_or_else(|error| panic!("expected {} to exist: {error}", rotated.display())) + .len(); + assert!( + size >= EVENT_LOG_ROTATE_BYTES, + "expected the whole capped log in {}, found {size} bytes", + rotated.display() + ); + assert!( + audit_records(&rotated) + .iter() + .all(|record| record.action == SEEDED_AUDIT_ACTION), + "the rotated generation must hold exactly the seeded records" + ); + assert!( + !live.with_file_name("events.jsonl.2").exists(), + "a single rollover makes a single rotated generation" + ); +} + +#[then("the permissions change starts a fresh audit log")] +async fn fresh_audit_log(world: &mut E2eWorld) { + let records = audit_records(&world.app_paths().audit_events_path()); + let actions: Vec<&str> = records + .iter() + .map(|record| record.action.as_str()) + .collect(); + assert_eq!( + actions, + ["set_mode"], + "expected the live audit log to hold only the permissions change" + ); +} + +#[then("the recent background activity still includes entries from before the rotation")] +async fn recent_activity_spans_rotation(world: &mut E2eWorld) { + let out = ok_output(world); + let Some((_, activity)) = out.split_once("recent background activity:") else { + panic!("expected a recent background activity section:\n{out}"); + }; + // Five most recent: four seeded records, which now live only in the + // rotated generation, then the permissions change. + assert_eq!( + activity.matches("Install activity was recorded.").count(), + 4, + "expected the four newest seeded records from the rotated log:\n{out}" + ); + assert_eq!( + activity + .matches("Background activity was recorded.") + .count(), + 1, + "expected the permissions change from the fresh log:\n{out}" + ); +} + +#[then("each rotated generation holds what the previous one held")] +async fn chain_shifted_by_one(world: &mut E2eWorld) { + ok_output(world); + let live = world.app_paths().audit_events_path(); + assert!( + audit_records(&live.with_file_name("events.jsonl.1")) + .iter() + .all(|record| record.action == SEEDED_LIVE_BEFORE_CHAIN_ROTATION), + "expected .1 to hold only what was the live file before this rotation" + ); + for generation in 2..=EVENT_LOG_ROTATED_GENERATIONS { + let actions: Vec = + audit_records(&live.with_file_name(format!("events.jsonl.{generation}"))) + .into_iter() + .map(|record| record.action) + .collect(); + assert_eq!( + actions, + [seeded_generation_action(generation - 1)], + "expected .{generation} to hold what .{} held before this rotation", + generation - 1 + ); + } +} + +#[then("the oldest previous generation is gone")] +async fn oldest_generation_dropped(world: &mut E2eWorld) { + let live = world.app_paths().audit_events_path(); + assert!( + !live + .with_file_name(format!( + "events.jsonl.{}", + EVENT_LOG_ROTATED_GENERATIONS + 1 + )) + .exists(), + "rotation must never keep more than {EVENT_LOG_ROTATED_GENERATIONS} generations" + ); + let dropped_action = seeded_generation_action(EVENT_LOG_ROTATED_GENERATIONS); + for generation in 1..=EVENT_LOG_ROTATED_GENERATIONS { + assert!( + !audit_records(&live.with_file_name(format!("events.jsonl.{generation}"))) + .iter() + .any(|record| record.action == dropped_action), + "the oldest generation's content ({dropped_action}) must not survive in .{generation}" + ); + } +} + +#[then("the CLI confirms the approval")] +async fn confirms_approval(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains(&format!("Proposal {NEW_PENDING_PROPOSAL} approved.")), + "expected an approval confirmation, got:\n{out}" + ); +} + +#[then("only the most recently reviewed proposals are kept")] +async fn reviewed_history_compacted(world: &mut E2eWorld) { + let proposals = + load_automation_proposals(&world.app_paths()).expect("proposal history reads back"); + let reviewed: Vec<&str> = proposals + .iter() + .filter(|proposal| proposal.status != "pending") + .map(|proposal| proposal.proposal_id.as_str()) + .collect(); + assert_eq!( + reviewed.len(), + REVIEWED_PROPOSALS_RETAINED, + "expected the review history capped at {REVIEWED_PROPOSALS_RETAINED}" + ); + // The approval just made is the newest review, so the oldest reviews give + // way to it: the excess plus one. + for n in 0..=EXCESS_REVIEWED { + let dropped = format!("e2e-reviewed-{n}"); + assert!( + !reviewed.contains(&dropped.as_str()), + "expected {dropped} dropped" + ); + } + assert_eq!(reviewed.last(), Some(&NEW_PENDING_PROPOSAL)); +} + +#[then("the CLI shows it still pending")] +async fn shows_still_pending(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains(&format!("Proposal {OLD_PENDING_PROPOSAL} (pending)")), + "expected the older pending proposal to still be found, got:\n{out}" + ); +} + // ── Helpers ──────────────────────────────────────────────────────── fn record(world: &mut E2eWorld, stdout: String, stderr: String, rc: i32) {