From afdb4227003f4c31d931fd4e8fb66a7820d775b6 Mon Sep 17 00:00:00 2001 From: Vincent Date: Sat, 7 Nov 2026 18:05:44 -0500 Subject: [PATCH 1/9] perf: share one span render context per transaction Every caller of retrieve_span_if_present built a fresh SpanRenderContext, which verifies and decodes the whole visible closure, for every file it rendered. On a span import of hyperfine (1,097 commits, 75 files) one `atomic view switch` built the context 312 times, `atomic record -a` 466 times and `atomic restore -f` 157 times, at roughly 55 ms each, while the materialization itself took milliseconds. - Add SharedSpanRender: a lazily built context shared by every file read through one transaction and visibility closure. The per-file root, capability and visibility checks are unchanged; only the closure decode is shared. - Use it in project_tree, project_change_closure_with_txn and plan_file_moves, which render every projected file in one transaction. - Add Repository::get_file_contents_on_view and recorded_presence_on_view so callers that read many paths get one transaction, one projection and one context, with the absent-liveness check scoped to exactly those paths. Use them in the shim checkpoint observation, working-copy verification and the CLI and daemon restore paths, which previously rendered (and discarded) every file one at a time. Batched reads are tested byte for byte against the single-path reads for graph-origin, span-rooted, empty, directory and absent paths on two views. On the hyperfine span import, `atomic view switch` drops from 25-34 s to 4-7 s (12 context builds) and `atomic restore -f` of all 75 files from 15 s to 4.8 s (8 builds); graph-only repositories are unaffected. --- atomic-cli/src/commands/git/checkpoint.rs | 20 +- atomic-cli/src/commands/restore.rs | 123 +++++++++- atomic-repository/src/repository/content.rs | 221 ++++++++++++++++-- .../src/repository/deferred_tree.rs | 27 ++- atomic-repository/src/repository/mod.rs | 1 + .../src/repository/project_tree.rs | 22 +- atomic-repository/src/repository/record.rs | 8 +- .../repository/tests/batch_content_tests.rs | 123 ++++++++++ atomic-repository/src/repository/tests/mod.rs | 1 + atomic-repository/src/repository/verify.rs | 23 +- libatomic/src/daemon/services.rs | 36 ++- 11 files changed, 546 insertions(+), 59 deletions(-) create mode 100644 atomic-repository/src/repository/tests/batch_content_tests.rs diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index 26b675e4..df7db7af 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -344,21 +344,23 @@ fn observe_atomic_on_handle( .map_err(|error| CheckpointError::Atomic(error.to_string()))? .map(|(object, markers)| (object.hash().ok(), markers)); + // One read for every path: the view is projected and its span history + // decoded once instead of once per file. + let contents = repo + .get_file_contents_on_view(&paths, view) + .map_err(|error| CheckpointError::Atomic(error.to_string()))?; let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for path in paths { + for (path, graph_content) in paths.into_iter().zip(contents) { let content = match &conflict_markers { Some((_, markers)) if markers.contains_key(&path) => { markers.get(&path).cloned().expect("marker bytes") } - _ => repo - .get_file_content_on_view(&path, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))? - .ok_or_else(|| { - CheckpointError::Atomic(format!( - "visible path '{path}' has no graph-derived content on view '{view}'" - )) - })?, + _ => graph_content.ok_or_else(|| { + CheckpointError::Atomic(format!( + "visible path '{path}' has no graph-derived content on view '{view}'" + )) + })?, }; hasher.update(&(path.len() as u64).to_le_bytes()); hasher.update(path.as_bytes()); diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 544db13e..75c5cded 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -59,7 +59,7 @@ use std::path::{Path, PathBuf}; use atomic_repository::tracking::TrackingOptions; use atomic_repository::{ - FileStatus, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, + FileStatus, RecordedPresence, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, }; use clap::Parser; @@ -247,6 +247,106 @@ impl Restore { } } + /// Restore every path that the view projects as a file, in one + /// materialization. Returns `(restored, failed)` counts. + /// + /// Paths the view does not project (or projects as a directory) are + /// skipped silently, and a path with an unresolved name conflict is a + /// per-file failure, exactly as the single-file path reports them. If the + /// batched materialization itself fails, each path falls back to the + /// single-file restore so the per-file outcome stays accurate. + fn restore_contents( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> (usize, usize) { + if paths.is_empty() { + return (0, 0); + } + let names: Vec = paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let presence = match repo.recorded_presence_on_view(&names, view) { + Ok(presence) => presence, + Err(error) => { + print_warning(&format!("Failed to project the working copy: {}", error)); + return self.restore_contents_one_by_one(repo, working_copy, view, paths); + } + }; + let mut failed = 0; + let mut selected = Vec::new(); + for (name, presence) in names.iter().zip(presence) { + match presence { + RecordedPresence::Present => selected.push(name.clone()), + RecordedPresence::Absent => {} + RecordedPresence::NameConflict => { + print_warning(&format!( + "Failed to restore '{}': path has an unresolved name conflict; refusing to choose content", + name + )); + failed += 1; + } + } + } + if selected.is_empty() { + return (0, failed); + } + match repo.materialize_paths(working_copy, selected.iter().cloned().collect()) { + Ok(_) => { + for name in &selected { + println!(" Restored: {}", name); + } + (selected.len(), failed) + } + Err(error) => { + print_warning(&format!( + "Batched restore failed ({}); retrying file by file", + error + )); + let retry: Vec = selected.iter().map(PathBuf::from).collect(); + let (restored, retry_failed) = + self.restore_contents_one_by_one(repo, working_copy, view, &retry); + (restored, failed + retry_failed) + } + } + } + + fn restore_contents_one_by_one( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> (usize, usize) { + let mut restored = 0; + let mut failed = 0; + for path in paths { + let path_display = path.display(); + match self.restore_file( + repo, + working_copy, + view, + Path::new(""), + path, + FileStatus::Modified, + ) { + Ok(RestoreOutcome::Restored) => { + println!(" Restored: {}", path_display); + restored += 1; + } + Ok(RestoreOutcome::Untracked | RestoreOutcome::Skipped) => {} + Err(e) => { + print_warning(&format!("Failed to restore '{}': {}", path_display, e)); + failed += 1; + } + } + } + (restored, failed) + } + /// Restore a single file according to its status. /// /// - `Added` (tracked but not recorded): undo tracking. The file stays @@ -455,9 +555,16 @@ impl Command for Restore { let mut restored_count = 0; let mut error_count = 0; + // Added files are untracked one by one; every other dirty file is + // restored in a single materialization so the view is projected and + // its span history decoded once, not once per file. + let mut content_paths = Vec::new(); for (path, file_status) in &files_to_restore { + if *file_status != FileStatus::Added { + content_paths.push(path.clone()); + continue; + } let path_display = path.display(); - match self.restore_file( &repo, working_copy, @@ -466,23 +573,21 @@ impl Command for Restore { path, *file_status, ) { - Ok(RestoreOutcome::Restored) => { - println!(" Restored: {}", path_display); - restored_count += 1; - } Ok(RestoreOutcome::Untracked) => { println!(" Untracked: {} (kept on disk)", path_display); restored_count += 1; } - Ok(RestoreOutcome::Skipped) => { - // No pristine content to restore; nothing to do. - } + Ok(RestoreOutcome::Restored | RestoreOutcome::Skipped) => {} Err(e) => { print_warning(&format!("Failed to restore '{}': {}", path_display, e)); error_count += 1; } } } + let (restored, failed) = + self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); + restored_count += restored; + error_count += failed; // Summary println!(); diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index fa03ff60..b0e93b14 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -23,6 +23,17 @@ fn graph_visibility_at_change( graph_visibility_from_membership(txn, &membership).map(Some) } +/// Whether a view projects a file at a path, without rendering its content. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecordedPresence { + /// The view projects a non-directory entry at this path. + Present, + /// The path is absent on the view, or projects a directory. + Absent, + /// The path has an unresolved name conflict; no single content applies. + NameConflict, +} + impl Repository { /// Get the recorded content for a tracked file. /// @@ -289,6 +300,49 @@ impl Repository { } } + /// Decide, for many paths at once, whether a view projects a file there. + /// + /// One transaction and one tree projection answer every path; nothing is + /// rendered. The rules match [`Self::get_materialized_entry_on_view`]: + /// a visible name conflict is reported instead of choosing content, and a + /// directory or an absent path is `Absent`. + pub fn recorded_presence_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result, RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + Ok(paths + .iter() + .map(|path| { + let normalized = normalize_path(Path::new(path)); + if projection.name_conflicts.contains_key(&normalized) { + return RecordedPresence::NameConflict; + } + match projection.present.get(&normalized) { + Some(item) if !item.is_directory => RecordedPresence::Present, + _ => RecordedPresence::Absent, + } + }) + .collect()) + } + /// Resolve a file as an explicit present/absent materialization entry on a view. /// /// Presence comes from the operation-aware path lifecycle projection. Empty @@ -335,22 +389,66 @@ impl Repository { if item.is_directory { return Ok(MaterializedEntry::absent(normalized, Some(item.inode))); } - let bytes = - match retrieve_span_if_present(&txn, &self.change_store, item.inode, &visibility)? { - Some(bytes) => bytes, - None => retrieve_content_with_filter_fast( - &txn, - &self.change_store, - item.inode, - item.position, - atomic_core::output::alive::RetrieveOptions::new() - .with_graph_visibility(visibility), - ) - .map_err(|error| RepositoryError::Database(error.to_string()))?, - }; + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + let bytes = render_projected_file(&txn, &self.change_store, &spans, item, &visibility)?; Ok(MaterializedEntry::present(normalized, item.inode, bytes)) } + /// Render many files on a view through one transaction, one tree + /// projection and one shared span context. + /// + /// Each entry is exactly what [`Self::get_file_content_on_view`] returns + /// for that path: `None` for an absent path or a directory. A requested + /// path with an unresolved name conflict fails the whole call, as the + /// single-path read refuses to choose content for it. + pub fn get_file_contents_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result>>, RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + paths + .iter() + .map(|path| { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return Err(RepositoryError::InvalidOperation { + message: format!( + "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", + normalized, + conflict.paths.len(), + conflict.sides.len() + ), + }); + } + match projection.present.get(&normalized) { + Some(item) if !item.is_directory => { + render_projected_file(&txn, &self.change_store, &spans, item, &visibility) + .map(Some) + } + _ => Ok(None), + } + }) + .collect() + } + /// Read a path through an exact assembly visibility: `view`'s validated /// closure minus `excluded` changes (CB-9B review R2). /// @@ -907,6 +1005,37 @@ impl Repository { } } +/// Recorded bytes of one projected file: its span rendering when the file is +/// span-rooted on this view, otherwise the filtered graph rendering. +fn render_projected_file( + txn: &T, + store: &C, + spans: &SharedSpanRender<'_, T, C>, + item: &atomic_core::output::repo::OutputItem, + visibility: &GraphVisibilityClosure, +) -> Result, RepositoryError> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::GraphTxnT + + atomic_core::pristine::InodeGraphOps + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + if let Some(bytes) = spans.render_if_present(item.inode)? { + return Ok(bytes); + } + retrieve_content_with_filter_fast( + txn, + store, + item.inode, + item.position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(visibility.clone()), + ) + .map_err(|error| RepositoryError::Database(error.to_string())) +} + pub(super) fn retrieve_span_if_present( txn: &T, store: &C, @@ -928,6 +1057,72 @@ where .map_err(|error| RepositoryError::Database(error.to_string())) } +/// A span renderer shared by every file read through one transaction and +/// visibility closure. +/// +/// Building a [`SpanRenderContext`] verifies and decodes the whole visible +/// closure, so callers that render many files in one transaction must not +/// rebuild it per file. The context is built lazily on the first span-rooted +/// file and reused for the rest; graph-origin files never trigger a build. +/// The per-file root, capability and visibility checks are unchanged. +pub(super) struct SharedSpanRender< + 'a, + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn, + C, +> { + txn: &'a T, + store: &'a C, + visibility: &'a GraphVisibilityClosure, + context: + std::cell::OnceCell>, +} + +impl<'a, T, C> SharedSpanRender<'a, T, C> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + pub(super) fn new(txn: &'a T, store: &'a C, visibility: &'a GraphVisibilityClosure) -> Self { + Self { + txn, + store, + visibility, + context: std::cell::OnceCell::new(), + } + } + + /// Render `inode` through the shared context when its span root is visible; + /// `None` means the file is graph-origin and must be read from the graph. + pub(super) fn render_if_present( + &self, + inode: Inode, + ) -> Result>, RepositoryError> { + if !has_visible_span_root(self.txn, self.store, inode, self.visibility)? { + return Ok(None); + } + let context = match self.context.get() { + Some(context) => context, + None => { + let built = atomic_core::pristine::span_patch_storage::SpanRenderContext::new( + self.txn, + self.store, + self.visibility, + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + // A concurrent `set` is impossible: `OnceCell` is single-threaded. + let _ = self.context.set(built); + self.context.get().expect("span context was just set") + } + }; + context + .render(inode) + .map(Some) + .map_err(|error| RepositoryError::Database(error.to_string())) + } +} + pub(super) fn retrieve_span_bounded_if_present( txn: &T, store: &C, diff --git a/atomic-repository/src/repository/deferred_tree.rs b/atomic-repository/src/repository/deferred_tree.rs index cc2482ae..3b7725e6 100644 --- a/atomic-repository/src/repository/deferred_tree.rs +++ b/atomic-repository/src/repository/deferred_tree.rs @@ -1051,6 +1051,31 @@ impl Repository { visibility: &GraphVisibilityClosure, wanted: Option<&str>, ) -> Result + where + T: GraphTxnT + + TreeTxnT + + PathClaimTxnT + + atomic_core::pristine::InodeGraphOps, + { + match wanted { + Some(path) => self.project_tree_for_visibility_wanted( + txn, + visibility, + Some(&HashSet::from([path.to_string()])), + ), + None => self.project_tree_for_visibility_wanted(txn, visibility, None), + } + } + + /// Set-scoped variant of [`Self::project_tree_for_visibility_scoped`]: + /// the absent-liveness check runs only for the `wanted` paths, so a + /// batch of path reads costs what the same reads cost one by one. + pub(super) fn project_tree_for_visibility_wanted( + &self, + txn: &T, + visibility: &GraphVisibilityClosure, + wanted: Option<&HashSet>, + ) -> Result where T: GraphTxnT + TreeTxnT @@ -1105,7 +1130,7 @@ impl Repository { let mut absent_check_ms = 0u128; let visible_changes = visibility.iter_dependency_first().copied().collect(); for absent in reduced.absent { - let wanted_here = wanted.is_none_or(|path| path == absent.path.as_str()); + let wanted_here = wanted.is_none_or(|paths| paths.contains(absent.path.as_str())); if wanted_here && !absent.directory && !alive_inodes.contains(&absent.inode) diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 51f727eb..863a0c94 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -44,6 +44,7 @@ use std::sync::Arc; use atomic_core::change::{Change, ChangeHeader, GraphOp}; pub use atomic_core::output::repo::MaterializedEntry; use atomic_core::output::repo::{MaterializeResult, OutputItem}; +pub use content::RecordedPresence; use atomic_core::pristine::{ GraphTxnT, GraphVisibilityClosure, MutTxnT, Pristine, TreeTxnT, ViewMembershipSet, ViewScope, diff --git a/atomic-repository/src/repository/project_tree.rs b/atomic-repository/src/repository/project_tree.rs index 1a945254..62a54fd6 100644 --- a/atomic-repository/src/repository/project_tree.rs +++ b/atomic-repository/src/repository/project_tree.rs @@ -1229,6 +1229,7 @@ impl Repository { ))); } let mut entries = Vec::new(); + let spans = super::content::SharedSpanRender::new(&txn, &self.change_store, &closure); for item in projection .present .values() @@ -1261,13 +1262,9 @@ impl Repository { ))); } let materialization = attrs.materialization; - let bytes = match super::content::retrieve_span_if_present( - &txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + let bytes = match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( @@ -1445,6 +1442,7 @@ impl Repository { } let mut entries = Vec::new(); let mut projected_paths = std::collections::BTreeSet::new(); + let spans = super::content::SharedSpanRender::new(txn, &self.change_store, &closure); for item in projection .present .values() @@ -1468,13 +1466,9 @@ impl Repository { let materialization = attrs.materialization; let bytes = match marker_bytes.get(item.path.as_str()) { Some(override_bytes) => override_bytes.clone(), - None => match super::content::retrieve_span_if_present( - txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + None => match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index c4e13d0b..f968b06c 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -4694,11 +4694,13 @@ fn plan_file_moves( basis: MoveBasis, } + // One span context serves every deleted candidate in this planning pass. + let spans = super::content::SharedSpanRender::new(cached, store, visibility); let load_old_content = |inode: Inode, position: Position| -> Result, RecordError> { - if let Some(bytes) = - super::content::retrieve_span_if_present(cached, store, inode, visibility) - .map_err(RecordError::Repository)? + if let Some(bytes) = spans + .render_if_present(inode) + .map_err(RecordError::Repository)? { return Ok(bytes); } diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs new file mode 100644 index 00000000..0fc023de --- /dev/null +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -0,0 +1,123 @@ +use super::*; +use atomic_core::pristine::{CapabilityMutTxnT, MutTxnT, SPAN_RANGE_CAPABILITY}; + +fn record_all(repo: &TestRepository, message: &str) { + repo.record( + ChangeHeader::new(message), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true) + .sync_vault(false) + .enrich_kg(false), + ) + .unwrap(); +} + +fn enable_spans(repo: &TestRepository) { + let mut txn = repo.pristine().write_txn().unwrap(); + txn.put_required_capability(SPAN_RANGE_CAPABILITY).unwrap(); + txn.commit().unwrap(); +} + +fn write_and_add(temp: &tempfile::TempDir, repo: &mut TestRepository, path: &str, bytes: &[u8]) { + let full = temp.path().join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(full, bytes).unwrap(); + repo.add(path, TrackingOptions::default()).unwrap(); +} + +/// Batched reads must be byte-identical to the single-path reads they replace, +/// for graph-origin files, span-rooted files, empty files, directories and +/// absent paths, on every view they are asked about. +#[test] +fn batch_reads_and_presence_match_single_path_reads() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "legacy.txt", b"legacy\n"); + record_all(&repo, "graph origin"); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "empty.txt", b""); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "new files"); + std::fs::write(temp.path().join("b.txt"), b"beta edited\n").unwrap(); + record_all(&repo, "edit b"); + + repo.create_view_from("feature", "dev").unwrap(); + repo.switch_view("feature").unwrap(); + std::fs::write(temp.path().join("a.txt"), b"alpha on feature\n").unwrap(); + record_all(&repo, "feature edit"); + + let paths: Vec = [ + "a.txt", + "b.txt", + "empty.txt", + "legacy.txt", + "dir/c.txt", + "dir", + "missing.txt", + ] + .map(str::to_string) + .into(); + for view in ["dev", "feature"] { + let batch = repo.get_file_contents_on_view(&paths, view).unwrap(); + assert_eq!(batch.len(), paths.len()); + for (path, bytes) in paths.iter().zip(&batch) { + assert_eq!( + bytes, + &repo.get_file_content_on_view(path, view).unwrap(), + "spans={spans} view={view} path={path}" + ); + } + let expected_a: &[u8] = if view == "feature" { + b"alpha on feature\n" + } else { + b"alpha\n" + }; + assert_eq!(batch[0].as_deref(), Some(expected_a)); + assert_eq!(batch[1].as_deref(), Some(b"beta edited\n".as_slice())); + assert_eq!(batch[2].as_deref(), Some(b"".as_slice())); + assert_eq!(batch[3].as_deref(), Some(b"legacy\n".as_slice())); + assert_eq!(batch[4].as_deref(), Some(b"gamma\n".as_slice())); + assert_eq!(batch[5], None, "directories have no content"); + assert_eq!(batch[6], None, "absent paths have no content"); + + let presence = repo.recorded_presence_on_view(&paths, view).unwrap(); + let expected: Vec = batch + .iter() + .map(|bytes| { + if bytes.is_some() { + RecordedPresence::Present + } else { + RecordedPresence::Absent + } + }) + .collect(); + assert_eq!(presence, expected, "spans={spans} view={view}"); + } + } +} + +/// An unknown view fails the batch exactly like the single-path read. +#[test] +fn batch_reads_reject_unknown_views() { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + record_all(&repo, "one file"); + let paths = vec!["a.txt".to_string()]; + assert!(matches!( + repo.get_file_contents_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert!(matches!( + repo.recorded_presence_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert_eq!(repo.get_file_contents_on_view(&[], "dev").unwrap(), vec![]); +} diff --git a/atomic-repository/src/repository/tests/mod.rs b/atomic-repository/src/repository/tests/mod.rs index 80d3a470..0a505b6f 100644 --- a/atomic-repository/src/repository/tests/mod.rs +++ b/atomic-repository/src/repository/tests/mod.rs @@ -9,6 +9,7 @@ use std::path::Path; use tempfile::TempDir; mod attribute_tests; +mod batch_content_tests; mod binding_fetch_tests; mod binding_store_tests; mod binding_verify_tests; diff --git a/atomic-repository/src/repository/verify.rs b/atomic-repository/src/repository/verify.rs index b41df533..18554ba5 100644 --- a/atomic-repository/src/repository/verify.rs +++ b/atomic-repository/src/repository/verify.rs @@ -138,6 +138,27 @@ impl Repository { // Files visible (tracked + recorded) on the current view. let visible = self.visible_file_paths(&desired_view)?; + // Clean files are compared against the graph; read them all through + // one transaction so the view's span history is decoded once. + let is_uncommitted_or_conflicted = |path: &String| { + matches!( + status_by_path.get(path), + Some(FileStatus::Modified) + | Some(FileStatus::Added) + | Some(FileStatus::Deleted) + | Some(FileStatus::Conflicted) + ) + }; + let clean_paths: Vec = visible + .iter() + .filter(|path| !is_uncommitted_or_conflicted(path)) + .cloned() + .collect(); + let graph_contents: std::collections::HashMap<&String, Option>> = clean_paths + .iter() + .zip(self.get_file_contents_on_view(&clean_paths, &desired_view)?) + .collect(); + for path in &visible { let st = status_by_path.get(path).copied(); @@ -156,7 +177,7 @@ impl Repository { _ => { let abs = self.root.join(path); let disk = std::fs::read(&abs).ok(); - let graph = self.get_file_content_on_view(path, &desired_view)?; + let graph = graph_contents.get(path).cloned().flatten(); if let (Some(disk), Some(graph)) = (disk.as_ref(), graph.as_ref()) { report.clean_files_checked += 1; if disk != graph { diff --git a/libatomic/src/daemon/services.rs b/libatomic/src/daemon/services.rs index e7c69040..1431ed23 100644 --- a/libatomic/src/daemon/services.rs +++ b/libatomic/src/daemon/services.rs @@ -2045,6 +2045,7 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI } let mut restored: Vec = Vec::new(); let mut materialize = std::collections::HashSet::new(); + let mut content_paths: Vec = Vec::new(); for (path, file_status) in dirty { match file_status { // Added: undo the add — untrack, keep on disk. @@ -2058,15 +2059,32 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI restored.push(path); } // Modified/Deleted: pristine content back to disk. - _ => { - let content = repo - .get_file_content(std::path::Path::new(&path)) - .map_err(repository_error)?; - if content.is_some() { - materialize.insert(path.clone()); - restored.push(path); - } - // No pristine content: skip (nothing safe to do). + _ => content_paths.push(path), + } + } + // One projection decides which dirty paths the view still records; + // rendering every file here only to discard the bytes would decode + // the view's history once per file. + let presence = if content_paths.is_empty() { + Vec::new() + } else { + repo.recorded_presence_on_view(&content_paths, repo.current_view()) + .map_err(repository_error)? + }; + for (path, presence) in content_paths.into_iter().zip(presence) { + match presence { + atomic_repository::RecordedPresence::Present => { + materialize.insert(path.clone()); + restored.push(path); + } + // No pristine content: skip (nothing safe to do). + atomic_repository::RecordedPresence::Absent => {} + atomic_repository::RecordedPresence::NameConflict => { + return Err(repository_error(atomic_repository::RepositoryError::InvalidOperation { + message: format!( + "path '{path}' has an unresolved name conflict; refusing to choose content" + ), + })); } } } From b15406c2f6f98d3a267b6b07099087b66055b5c0 Mon Sep 17 00:00:00 2001 From: Vincent Date: Sat, 7 Nov 2026 19:45:44 -0500 Subject: [PATCH 2/9] fix: keep batched restore and checkpoint reads behaviour-identical Checkpoint observation no longer reads graph content for paths that carry conflict-marker bytes. Name-conflict errors carry the same path and claimant counts as the single-path reader. The daemon and CLI restore paths classify every path up front but still walk and report them in status order, so untracking, failures and output lines happen exactly where they did before. The batch-read test now asserts that the fixture really records span-rooted files. --- atomic-cli/src/commands/git/checkpoint.rs | 39 ++-- atomic-cli/src/commands/restore.rs | 185 ++++++++++-------- atomic-repository/src/repository/content.rs | 46 +++-- atomic-repository/src/repository/mod.rs | 2 +- .../repository/tests/batch_content_tests.rs | 16 ++ libatomic/src/daemon/services.rs | 76 ++++--- 6 files changed, 227 insertions(+), 137 deletions(-) diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index df7db7af..d16ba60b 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -344,19 +344,36 @@ fn observe_atomic_on_handle( .map_err(|error| CheckpointError::Atomic(error.to_string()))? .map(|(object, markers)| (object.hash().ok(), markers)); - // One read for every path: the view is projected and its span history - // decoded once instead of once per file. - let contents = repo - .get_file_contents_on_view(&paths, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))?; + // Paths with marker bytes never consult the graph. Every other path is + // read in one call so the view is projected and its span history decoded + // once instead of once per file. + let has_marker = |path: &String| -> bool { + conflict_markers + .as_ref() + .is_some_and(|(_, markers)| markers.contains_key(path)) + }; + let graph_paths: Vec = paths + .iter() + .filter(|path| !has_marker(path)) + .cloned() + .collect(); + let mut graph_contents: std::collections::HashMap>> = graph_paths + .iter() + .cloned() + .zip( + repo.get_file_contents_on_view(&graph_paths, view) + .map_err(|error| CheckpointError::Atomic(error.to_string()))?, + ) + .collect(); let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for (path, graph_content) in paths.into_iter().zip(contents) { - let content = match &conflict_markers { - Some((_, markers)) if markers.contains_key(&path) => { - markers.get(&path).cloned().expect("marker bytes") - } - _ => graph_content.ok_or_else(|| { + for path in paths { + let marker_bytes = conflict_markers + .as_ref() + .and_then(|(_, markers)| markers.get(&path).cloned()); + let content = match marker_bytes { + Some(marker_bytes) => marker_bytes, + None => graph_contents.remove(&path).flatten().ok_or_else(|| { CheckpointError::Atomic(format!( "visible path '{path}' has no graph-derived content on view '{view}'" )) diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 75c5cded..97fb0dad 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -55,6 +55,7 @@ //! Would restore: src/lib.rs //! ``` +use std::collections::HashMap; use std::path::{Path, PathBuf}; use atomic_repository::tracking::TrackingOptions; @@ -247,23 +248,27 @@ impl Restore { } } - /// Restore every path that the view projects as a file, in one - /// materialization. Returns `(restored, failed)` counts. + /// Classify and restore every Modified/Deleted path in one pass, returning + /// the outcome for each path so the caller can report them in status order. /// - /// Paths the view does not project (or projects as a directory) are - /// skipped silently, and a path with an unresolved name conflict is a - /// per-file failure, exactly as the single-file path reports them. If the - /// batched materialization itself fails, each path falls back to the - /// single-file restore so the per-file outcome stays accurate. + /// One projection decides which paths the view still records, and one + /// materialization writes all of them, instead of rendering and + /// materializing file by file. Outcomes match the single-file path: a path + /// the view does not project (or projects as a directory) is `Skipped`, + /// and a path with an unresolved name conflict fails with the same error. + /// A projection failure is reported for every path as the single-file read + /// would have reported it; if the batched materialization fails, every + /// path falls back to the single-file restore so outcomes stay per file. fn restore_contents( &self, repo: &Repository, working_copy: atomic_core::WorkingCopyId, view: &str, paths: &[PathBuf], - ) -> (usize, usize) { + ) -> HashMap> { + let mut outcomes = HashMap::new(); if paths.is_empty() { - return (0, 0); + return outcomes; } let names: Vec = paths .iter() @@ -272,46 +277,63 @@ impl Restore { let presence = match repo.recorded_presence_on_view(&names, view) { Ok(presence) => presence, Err(error) => { - print_warning(&format!("Failed to project the working copy: {}", error)); - return self.restore_contents_one_by_one(repo, working_copy, view, paths); + // A view or projection error would fail every single-path read + // the same way; report it per file without re-projecting. + for path in paths { + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); + } + return outcomes; } }; - let mut failed = 0; - let mut selected = Vec::new(); - for (name, presence) in names.iter().zip(presence) { + let mut selected: Vec<(PathBuf, String)> = Vec::new(); + for ((path, name), presence) in paths.iter().zip(&names).zip(presence) { match presence { - RecordedPresence::Present => selected.push(name.clone()), - RecordedPresence::Absent => {} - RecordedPresence::NameConflict => { - print_warning(&format!( - "Failed to restore '{}': path has an unresolved name conflict; refusing to choose content", - name - )); - failed += 1; + RecordedPresence::Present => selected.push((path.clone(), name.clone())), + RecordedPresence::Absent => { + outcomes.insert(path.clone(), Ok(RestoreOutcome::Skipped)); + } + RecordedPresence::NameConflict { paths, claimants } => { + let error = atomic_repository::name_conflict_error( + &atomic_repository::normalize_path(path), + paths, + claimants, + ); + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); } } } if selected.is_empty() { - return (0, failed); + return outcomes; } - match repo.materialize_paths(working_copy, selected.iter().cloned().collect()) { + let batch = selected.iter().map(|(_, name)| name.clone()).collect(); + match repo.materialize_paths(working_copy, batch) { Ok(_) => { - for name in &selected { - println!(" Restored: {}", name); + for (path, _) in selected { + outcomes.insert(path, Ok(RestoreOutcome::Restored)); } - (selected.len(), failed) } Err(error) => { print_warning(&format!( - "Batched restore failed ({}); retrying file by file", + "Batched restore failed ({}); restoring file by file", error )); - let retry: Vec = selected.iter().map(PathBuf::from).collect(); - let (restored, retry_failed) = - self.restore_contents_one_by_one(repo, working_copy, view, &retry); - (restored, failed + retry_failed) + let retry: Vec = selected.into_iter().map(|(path, _)| path).collect(); + outcomes.extend(self.restore_contents_one_by_one(repo, working_copy, view, &retry)); } } + outcomes } fn restore_contents_one_by_one( @@ -320,31 +342,21 @@ impl Restore { working_copy: atomic_core::WorkingCopyId, view: &str, paths: &[PathBuf], - ) -> (usize, usize) { - let mut restored = 0; - let mut failed = 0; - for path in paths { - let path_display = path.display(); - match self.restore_file( - repo, - working_copy, - view, - Path::new(""), - path, - FileStatus::Modified, - ) { - Ok(RestoreOutcome::Restored) => { - println!(" Restored: {}", path_display); - restored += 1; - } - Ok(RestoreOutcome::Untracked | RestoreOutcome::Skipped) => {} - Err(e) => { - print_warning(&format!("Failed to restore '{}': {}", path_display, e)); - failed += 1; - } - } - } - (restored, failed) + ) -> HashMap> { + paths + .iter() + .map(|path| { + let outcome = self.restore_file( + repo, + working_copy, + view, + Path::new(""), + path, + FileStatus::Modified, + ); + (path.clone(), outcome) + }) + .collect() } /// Restore a single file according to its status. @@ -555,39 +567,56 @@ impl Command for Restore { let mut restored_count = 0; let mut error_count = 0; - // Added files are untracked one by one; every other dirty file is - // restored in a single materialization so the view is projected and - // its span history decoded once, not once per file. - let mut content_paths = Vec::new(); + // Modified/Deleted files are classified and materialized together so + // the view is projected and its span history decoded once, not once + // per file. Added files are untracked one by one. Outcomes are + // reported in status order, as before. + let content_paths: Vec = files_to_restore + .iter() + .filter(|(_, file_status)| *file_status != FileStatus::Added) + .map(|(path, _)| path.clone()) + .collect(); + let mut content_outcomes = + self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); + for (path, file_status) in &files_to_restore { - if *file_status != FileStatus::Added { - content_paths.push(path.clone()); - continue; - } let path_display = path.display(); - match self.restore_file( - &repo, - working_copy, - &workspace_view, - &repo_root, - path, - *file_status, - ) { + + let outcome = if *file_status == FileStatus::Added { + self.restore_file( + &repo, + working_copy, + &workspace_view, + &repo_root, + path, + *file_status, + ) + } else { + content_outcomes.remove(path).unwrap_or_else(|| { + Err(CliError::Internal(anyhow::anyhow!( + "no restore outcome recorded for '{}'", + path_display + ))) + }) + }; + match outcome { + Ok(RestoreOutcome::Restored) => { + println!(" Restored: {}", path_display); + restored_count += 1; + } Ok(RestoreOutcome::Untracked) => { println!(" Untracked: {} (kept on disk)", path_display); restored_count += 1; } - Ok(RestoreOutcome::Restored | RestoreOutcome::Skipped) => {} + Ok(RestoreOutcome::Skipped) => { + // No pristine content to restore; nothing to do. + } Err(e) => { print_warning(&format!("Failed to restore '{}': {}", path_display, e)); error_count += 1; } } } - let (restored, failed) = - self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); - restored_count += restored; - error_count += failed; // Summary println!(); diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index b0e93b14..ecc4f1c0 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -31,7 +31,18 @@ pub enum RecordedPresence { /// The path is absent on the view, or projects a directory. Absent, /// The path has an unresolved name conflict; no single content applies. - NameConflict, + NameConflict { paths: usize, claimants: usize }, +} + +/// The error every content read returns for a path with an unresolved name +/// conflict. Callers that classify paths first must report the same error. +pub fn name_conflict_error(path: &str, paths: usize, claimants: usize) -> RepositoryError { + RepositoryError::InvalidOperation { + message: format!( + "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", + path, paths, claimants + ), + } } impl Repository { @@ -332,8 +343,11 @@ impl Repository { .iter() .map(|path| { let normalized = normalize_path(Path::new(path)); - if projection.name_conflicts.contains_key(&normalized) { - return RecordedPresence::NameConflict; + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return RecordedPresence::NameConflict { + paths: conflict.paths.len(), + claimants: conflict.sides.len(), + }; } match projection.present.get(&normalized) { Some(item) if !item.is_directory => RecordedPresence::Present, @@ -369,14 +383,11 @@ impl Repository { let projection = self.project_tree_for_visibility_scoped(&txn, &visibility, Some(normalized.as_str()))?; if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(RepositoryError::InvalidOperation { - message: format!( - "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", - normalized, - conflict.paths.len(), - conflict.sides.len() - ), - }); + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); } let Some(item) = projection.present.get(&normalized) else { let inode = projection @@ -429,14 +440,11 @@ impl Repository { .map(|path| { let normalized = normalize_path(Path::new(path)); if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(RepositoryError::InvalidOperation { - message: format!( - "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", - normalized, - conflict.paths.len(), - conflict.sides.len() - ), - }); + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); } match projection.present.get(&normalized) { Some(item) if !item.is_directory => { diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 863a0c94..f5cb2fef 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -44,7 +44,7 @@ use std::sync::Arc; use atomic_core::change::{Change, ChangeHeader, GraphOp}; pub use atomic_core::output::repo::MaterializedEntry; use atomic_core::output::repo::{MaterializeResult, OutputItem}; -pub use content::RecordedPresence; +pub use content::{name_conflict_error, RecordedPresence}; use atomic_core::pristine::{ GraphTxnT, GraphVisibilityClosure, MutTxnT, Pristine, TreeTxnT, ViewMembershipSet, ViewScope, diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs index 0fc023de..7bca2d49 100644 --- a/atomic-repository/src/repository/tests/batch_content_tests.rs +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -54,6 +54,22 @@ fn batch_reads_and_presence_match_single_path_reads() { std::fs::write(temp.path().join("a.txt"), b"alpha on feature\n").unwrap(); record_all(&repo, "feature edit"); + if spans { + use atomic_core::pristine::span_patch_storage::SpanPatchReadTxn; + use atomic_core::pristine::TreeTxnT; + let txn = repo.pristine().read_txn().unwrap(); + let inode = txn.get_inode("a.txt").unwrap().unwrap(); + assert!( + txn.span_root(inode).unwrap().is_some(), + "fixture must record a.txt as a span-rooted file" + ); + let legacy = txn.get_inode("legacy.txt").unwrap().unwrap(); + assert!( + txn.span_root(legacy).unwrap().is_none(), + "fixture must keep legacy.txt graph-origin" + ); + } + let paths: Vec = [ "a.txt", "b.txt", diff --git a/libatomic/src/daemon/services.rs b/libatomic/src/daemon/services.rs index 1431ed23..ea937f63 100644 --- a/libatomic/src/daemon/services.rs +++ b/libatomic/src/daemon/services.rs @@ -2043,9 +2043,36 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI name paths explicitly or pass force", )); } + // One projection decides which Modified/Deleted paths the view + // still records; rendering every file here only to discard the + // bytes would decode the view's history once per file. A + // projection error therefore fails the request before any path is + // untracked, and a render error now surfaces from materialization + // (which plans before it writes). The walk below keeps status + // order, so untracking and a name-conflict failure happen at the + // same position as before. + let content_paths: Vec = dirty + .iter() + .filter(|(_, file_status)| { + *file_status != atomic_repository::status::FileStatus::Added + }) + .map(|(path, _)| path.clone()) + .collect(); + let presence: std::collections::HashMap = + if content_paths.is_empty() { + Default::default() + } else { + content_paths + .iter() + .cloned() + .zip( + repo.recorded_presence_on_view(&content_paths, repo.current_view()) + .map_err(repository_error)?, + ) + .collect() + }; let mut restored: Vec = Vec::new(); let mut materialize = std::collections::HashSet::new(); - let mut content_paths: Vec = Vec::new(); for (path, file_status) in dirty { match file_status { // Added: undo the add — untrack, keep on disk. @@ -2059,33 +2086,26 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI restored.push(path); } // Modified/Deleted: pristine content back to disk. - _ => content_paths.push(path), - } - } - // One projection decides which dirty paths the view still records; - // rendering every file here only to discard the bytes would decode - // the view's history once per file. - let presence = if content_paths.is_empty() { - Vec::new() - } else { - repo.recorded_presence_on_view(&content_paths, repo.current_view()) - .map_err(repository_error)? - }; - for (path, presence) in content_paths.into_iter().zip(presence) { - match presence { - atomic_repository::RecordedPresence::Present => { - materialize.insert(path.clone()); - restored.push(path); - } - // No pristine content: skip (nothing safe to do). - atomic_repository::RecordedPresence::Absent => {} - atomic_repository::RecordedPresence::NameConflict => { - return Err(repository_error(atomic_repository::RepositoryError::InvalidOperation { - message: format!( - "path '{path}' has an unresolved name conflict; refusing to choose content" - ), - })); - } + _ => match presence.get(&path) { + Some(atomic_repository::RecordedPresence::Present) => { + materialize.insert(path.clone()); + restored.push(path); + } + // No pristine content: skip (nothing safe to do). + Some(atomic_repository::RecordedPresence::Absent) | None => {} + Some(atomic_repository::RecordedPresence::NameConflict { + paths, + claimants, + }) => { + return Err(repository_error(atomic_repository::name_conflict_error( + &atomic_repository::tracking::normalize_path(std::path::Path::new( + &path, + )), + *paths, + *claimants, + ))); + } + }, } } if !materialize.is_empty() { From 84729e29c5fdcf60a261d998c2b7e245dcc62a2d Mon Sep 17 00:00:00 2001 From: Vincent Date: Mon, 9 Nov 2026 09:10:50 -0500 Subject: [PATCH 3/9] fix: stream checkpoint and verify file contents --- atomic-cli/src/commands/git/checkpoint.rs | 78 ++++++++++++------ atomic-repository/src/repository/content.rs | 81 ++++++++++++------- .../repository/tests/batch_content_tests.rs | 50 ++++++++++++ atomic-repository/src/repository/verify.rs | 41 +++++----- 4 files changed, 177 insertions(+), 73 deletions(-) diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index d16ba60b..02d98c25 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -345,44 +345,70 @@ fn observe_atomic_on_handle( .map(|(object, markers)| (object.hash().ok(), markers)); // Paths with marker bytes never consult the graph. Every other path is - // read in one call so the view is projected and its span history decoded - // once instead of once per file. - let has_marker = |path: &String| -> bool { + // streamed from one read (one projection, one span context) in the same + // sorted order, and hashed as it arrives so only one file is held at a + // time; marker paths are hashed from their marker bytes in between. + let marker_bytes = |path: &str| -> Option<&[u8]> { conflict_markers .as_ref() - .is_some_and(|(_, markers)| markers.contains_key(path)) + .and_then(|(_, markers)| markers.get(path)) + .map(Vec::as_slice) }; let graph_paths: Vec = paths .iter() - .filter(|path| !has_marker(path)) + .filter(|path| marker_bytes(path).is_none()) .cloned() .collect(); - let mut graph_contents: std::collections::HashMap>> = graph_paths - .iter() - .cloned() - .zip( - repo.get_file_contents_on_view(&graph_paths, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))?, - ) - .collect(); let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for path in paths { - let marker_bytes = conflict_markers - .as_ref() - .and_then(|(_, markers)| markers.get(&path).cloned()); - let content = match marker_bytes { - Some(marker_bytes) => marker_bytes, - None => graph_contents.remove(&path).flatten().ok_or_else(|| { - CheckpointError::Atomic(format!( - "visible path '{path}' has no graph-derived content on view '{view}'" - )) - })?, - }; + let hash_entry = |hasher: &mut blake3::Hasher, path: &str, content: &[u8]| { hasher.update(&(path.len() as u64).to_le_bytes()); hasher.update(path.as_bytes()); hasher.update(&(content.len() as u64).to_le_bytes()); - hasher.update(&content); + hasher.update(content); + }; + let mut next = 0usize; + let mut missing: Option = None; + let streamed = repo.for_each_file_content_on_view(&graph_paths, view, |path, bytes| { + while paths.get(next).is_some_and(|pending| pending != path) { + let pending = &paths[next]; + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + missing = Some(pending.clone()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{pending}' has no content"), + }); + } + } + next += 1; + } + let Some(content) = bytes else { + missing = Some(path.to_string()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{path}' has no content"), + }); + }; + hash_entry(&mut hasher, path, &content); + next += 1; + Ok(()) + }); + if let Some(path) = missing { + return Err(CheckpointError::Atomic(format!( + "visible path '{path}' has no graph-derived content on view '{view}'" + ))); + } + streamed.map_err(|error| CheckpointError::Atomic(error.to_string()))?; + while let Some(pending) = paths.get(next) { + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + return Err(CheckpointError::Atomic(format!( + "visible path '{pending}' has no graph-derived content on view '{view}'" + ))) + } + } + next += 1; } Ok(CheckpointAtomicState { diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index ecc4f1c0..34bee428 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -405,18 +405,23 @@ impl Repository { Ok(MaterializedEntry::present(normalized, item.inode, bytes)) } - /// Render many files on a view through one transaction, one tree - /// projection and one shared span context. - /// - /// Each entry is exactly what [`Self::get_file_content_on_view`] returns - /// for that path: `None` for an absent path or a directory. A requested - /// path with an unresolved name conflict fails the whole call, as the - /// single-path read refuses to choose content for it. - pub fn get_file_contents_on_view( + /// Stream the recorded bytes of many files on a view through one + /// transaction, one tree projection and one shared span context. + /// + /// `visit` receives each path in the order given, with exactly what + /// [`Self::get_file_content_on_view`] returns for it: `None` for an absent + /// path or a directory. Bytes are handed over one file at a time and + /// dropped by the visitor, so peak memory stays at one file, not the whole + /// set. A requested path with an unresolved name conflict fails the call, + /// as the single-path read refuses to choose content for it; a visitor + /// error stops the walk and is returned unchanged. Nothing survives this + /// call's transaction; the visitor must not mutate repository state. + pub fn for_each_file_content_on_view( &self, paths: &[String], view_name: &str, - ) -> Result>>, RepositoryError> { + mut visit: impl FnMut(&str, Option>) -> Result<(), RepositoryError>, + ) -> Result<(), RepositoryError> { let txn = self .pristine .read_txn() @@ -435,26 +440,44 @@ impl Repository { let projection = self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); - paths - .iter() - .map(|path| { - let normalized = normalize_path(Path::new(path)); - if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(name_conflict_error( - &normalized, - conflict.paths.len(), - conflict.sides.len(), - )); - } - match projection.present.get(&normalized) { - Some(item) if !item.is_directory => { - render_projected_file(&txn, &self.change_store, &spans, item, &visibility) - .map(Some) - } - _ => Ok(None), - } - }) - .collect() + for path in paths { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); + } + let bytes = match projection.present.get(&normalized) { + Some(item) if !item.is_directory => Some(render_projected_file( + &txn, + &self.change_store, + &spans, + item, + &visibility, + )?), + _ => None, + }; + visit(path, bytes)?; + } + Ok(()) + } + + /// Collected form of [`Self::for_each_file_content_on_view`]: one entry per + /// requested path, in order. This holds every rendered file at once, so + /// callers that only need to consume each file should stream instead. + pub fn get_file_contents_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result>>, RepositoryError> { + let mut contents = Vec::with_capacity(paths.len()); + self.for_each_file_content_on_view(paths, view_name, |_, bytes| { + contents.push(bytes); + Ok(()) + })?; + Ok(contents) } /// Read a path through an exact assembly visibility: `view`'s validated diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs index 7bca2d49..18903e9a 100644 --- a/atomic-repository/src/repository/tests/batch_content_tests.rs +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -137,3 +137,53 @@ fn batch_reads_reject_unknown_views() { )); assert_eq!(repo.get_file_contents_on_view(&[], "dev").unwrap(), vec![]); } + +/// The streaming read visits every path in the requested order with the same +/// bytes the collected read returns, and a visitor error stops the walk and +/// comes back unchanged. +#[test] +fn streaming_reads_visit_in_order_and_propagate_visitor_errors() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "files"); + let paths: Vec = ["dir/c.txt", "missing.txt", "a.txt", "dir", "b.txt"] + .map(str::to_string) + .into(); + let collected = repo.get_file_contents_on_view(&paths, "dev").unwrap(); + let mut visited = Vec::new(); + repo.for_each_file_content_on_view(&paths, "dev", |path, bytes| { + visited.push((path.to_string(), bytes)); + Ok(()) + }) + .unwrap(); + let visited_paths: Vec<&str> = visited.iter().map(|(path, _)| path.as_str()).collect(); + assert_eq!( + visited_paths, + paths.iter().map(String::as_str).collect::>() + ); + let visited_bytes: Vec>> = + visited.into_iter().map(|(_, bytes)| bytes).collect(); + assert_eq!(visited_bytes, collected, "spans={spans}"); + + let mut seen = Vec::new(); + let error = repo + .for_each_file_content_on_view(&paths, "dev", |path, _| { + seen.push(path.to_string()); + if path == "a.txt" { + return Err(RepositoryError::InvalidOperation { + message: "visitor refused".into(), + }); + } + Ok(()) + }) + .unwrap_err(); + assert!(error.to_string().contains("visitor refused"), "{error}"); + assert_eq!(seen, ["dir/c.txt", "missing.txt", "a.txt"], "spans={spans}"); + } +} diff --git a/atomic-repository/src/repository/verify.rs b/atomic-repository/src/repository/verify.rs index 18554ba5..67f6c92f 100644 --- a/atomic-repository/src/repository/verify.rs +++ b/atomic-repository/src/repository/verify.rs @@ -138,8 +138,10 @@ impl Repository { // Files visible (tracked + recorded) on the current view. let visible = self.visible_file_paths(&desired_view)?; - // Clean files are compared against the graph; read them all through - // one transaction so the view's span history is decoded once. + // Clean files are compared against the graph. Stream them through one + // transaction so the view's span history is decoded once, comparing + // each file as it arrives; only the mismatching lengths are kept and + // reported below in the visible-path order. let is_uncommitted_or_conflicted = |path: &String| { matches!( status_by_path.get(path), @@ -154,10 +156,19 @@ impl Repository { .filter(|path| !is_uncommitted_or_conflicted(path)) .cloned() .collect(); - let graph_contents: std::collections::HashMap<&String, Option>> = clean_paths - .iter() - .zip(self.get_file_contents_on_view(&clean_paths, &desired_view)?) - .collect(); + let mut drift: HashMap = HashMap::new(); + let mut clean_files_checked = 0usize; + self.for_each_file_content_on_view(&clean_paths, &desired_view, |path, graph| { + let disk = std::fs::read(self.root.join(path)).ok(); + if let (Some(disk), Some(graph)) = (disk, graph) { + clean_files_checked += 1; + if disk != graph { + drift.insert(path.to_string(), (disk.len(), graph.len())); + } + } + Ok(()) + })?; + report.clean_files_checked += clean_files_checked; for path in &visible { let st = status_by_path.get(path).copied(); @@ -175,18 +186,12 @@ impl Repository { // Clean (absent from status) or other benign states: the disk // must match the graph exactly. _ => { - let abs = self.root.join(path); - let disk = std::fs::read(&abs).ok(); - let graph = graph_contents.get(path).cloned().flatten(); - if let (Some(disk), Some(graph)) = (disk.as_ref(), graph.as_ref()) { - report.clean_files_checked += 1; - if disk != graph { - report.problems.push(VerifyProblem::MaterializationDrift { - path: path.clone(), - disk_len: disk.len(), - graph_len: graph.len(), - }); - } + if let Some((disk_len, graph_len)) = drift.remove(path.as_str()) { + report.problems.push(VerifyProblem::MaterializationDrift { + path: path.clone(), + disk_len, + graph_len, + }); } } } From e8fdb4f17bc8cd71e72f7997f53ae9f2896da4a2 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 9 Nov 2026 19:02:25 -0600 Subject: [PATCH 4/9] feat: partial materialization cases and Git working-copy baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add materialize_paths_noop, materialize_paths_missing and materialize_prefix_missing to the paired graph/span materialization benchmark, with per-sample subset write-count assertions, and a new git_baseline.py harness that times git checkout -- full working-tree restores on the same synthetic history shape. First paired run (macOS/aarch64, many-files, 8 iterations): partial materialize_paths restore of 8 of 25 files takes ~315 ms against ~565 ms full, while the partial noop floor (~187 ms) equals the full noop — the journaled plan/lease/journal path dominates, not content work. Span capability stays within 2% on every materialize case. Git restores the full tree in ~22 ms. --- .../examples/materialize_bench.rs | 77 ++++++++ tools/materialize-bench/README.md | 54 +++++ tools/materialize-bench/git_baseline.py | 184 ++++++++++++++++++ tools/materialize-bench/run.py | 1 + tools/materialize-bench/test_git_baseline.py | 61 ++++++ tools/materialize-bench/test_runner.py | 2 +- 6 files changed, 378 insertions(+), 1 deletion(-) create mode 100644 tools/materialize-bench/git_baseline.py create mode 100644 tools/materialize-bench/test_git_baseline.py diff --git a/atomic-repository/examples/materialize_bench.rs b/atomic-repository/examples/materialize_bench.rs index 39253d29..9a311003 100644 --- a/atomic-repository/examples/materialize_bench.rs +++ b/atomic-repository/examples/materialize_bench.rs @@ -21,12 +21,20 @@ const CASES: &[&str] = &[ "render_reopened", "materialize_noop", "materialize_missing", + "materialize_paths_noop", + "materialize_paths_missing", + "materialize_prefix_missing", "switch_one_forward", "switch_one_back", "switch_many_forward", "switch_many_back", ]; +/// Zero-based fixture index of a generated `files/file_NNNN.txt` path. +fn fixture_file_index(path: &str) -> Option { + path.strip_prefix("files/file_")?.get(..4)?.parse().ok() +} + #[derive(Clone, Debug, Serialize, Deserialize)] struct Config { output: PathBuf, @@ -151,6 +159,13 @@ struct Fixture { repo: Option, backend: &'static str, golden: BTreeMap>, + /// Every third generated file (including the edited hot file); the + /// `materialize_paths_*` partial subset. + partial_paths: std::collections::HashSet, + /// Filename prefix selecting the first ten generated files; the + /// `materialize_prefix_missing` subset. + prefix: String, + prefix_paths: std::collections::BTreeSet, setup_ns: u128, } @@ -240,11 +255,39 @@ impl Fixture { "fixture indexes unhealthy: {:?}", healthy.problems ); + let partial_paths: std::collections::HashSet = golden["dev"] + .iter() + .filter_map(|file| { + fixture_file_index(&file.path) + .filter(|index| index % 3 == 0) + .map(|_| file.path.clone()) + }) + .collect(); + ensure!( + !partial_paths.is_empty() && partial_paths.len() < golden["dev"].len(), + "degenerate partial path subset" + ); + let prefix = "files/file_000".to_string(); + let prefix_paths: std::collections::BTreeSet = golden["dev"] + .iter() + .filter_map(|file| { + fixture_file_index(&file.path) + .filter(|index| *index < 10) + .map(|_| file.path.clone()) + }) + .collect(); + ensure!( + !prefix_paths.is_empty() && prefix_paths.len() < golden["dev"].len(), + "degenerate prefix subset" + ); let fixture = Self { directory, repo: Some(repo), backend: if span { "span" } else { "graph" }, golden, + partial_paths, + prefix, + prefix_paths, setup_ns: started.elapsed().as_nanos(), }; fixture.verify("dev")?; @@ -362,9 +405,24 @@ fn sample(fixture: &mut Fixture, case: &str, iteration: usize) -> Result fs::remove_file(fixture.directory.path().join(&file.path))?; } } + if case == "materialize_paths_missing" || case == "materialize_prefix_missing" { + // Partial analog of materialize_missing: remove only the subset the + // partial call is expected to restore. + let subset: Vec<&String> = if case == "materialize_paths_missing" { + fixture.partial_paths.iter().collect() + } else { + fixture.prefix_paths.iter().collect() + }; + for path in subset { + fs::remove_file(fixture.directory.path().join(path))?; + } + } if case == "render_reopened" { drop(fixture.repo.take()); } + // Partial path sets are cloned outside the timed region. + let partial_paths = fixture.partial_paths.clone(); + let prefix = fixture.prefix.clone(); let started = Instant::now(); if case == "render_reopened" { let opened = Instant::now(); @@ -386,9 +444,28 @@ fn sample(fixture: &mut Fixture, case: &str, iteration: usize) -> Result } else { let result = if case.starts_with("switch_") { repo.switch_view(wc, target)? + } else if case == "materialize_paths_noop" || case == "materialize_paths_missing" { + repo.materialize_paths(wc, partial_paths)? + } else if case == "materialize_prefix_missing" { + repo.materialize_prefix(wc, &prefix)? } else { repo.materialize(wc)? }; + // A partial call must actually rewrite its subset, never report the + // whole view as written and never skip deleted files. + if case == "materialize_paths_missing" || case == "materialize_prefix_missing" { + let expected = if case == "materialize_paths_missing" { + fixture.partial_paths.len() + } else { + fixture.prefix_paths.len() + }; + ensure!( + result.files_written >= expected, + "partial materialize wrote {} files, expected at least {}", + result.files_written, + expected + ); + } written = Some(result.files_written); skipped = Some(result.files_skipped); bytes_written = Some(result.bytes_written); diff --git a/tools/materialize-bench/README.md b/tools/materialize-bench/README.md index d81aba2f..14b0da8a 100644 --- a/tools/materialize-bench/README.md +++ b/tools/materialize-bench/README.md @@ -38,6 +38,14 @@ Every fixture also includes an empty file. Override with `--files`, `--lines`, - `render_reopened`: drop/reopen the repository, then render; includes open cost. - `materialize_noop`: ordinary journaled materialization of already-correct files. - `materialize_missing`: same API after removing only generated tracked files. +- `materialize_paths_noop`: journaled `materialize_paths` over a fixed subset + (every third generated file, including the edited hot file) with nothing + deleted; measures the partial fast path when every file is already correct. +- `materialize_paths_missing`: the same subset after removing only those files; + the partial analog of `materialize_missing`. +- `materialize_prefix_missing`: remove the first ten generated files, then + journaled `materialize_prefix("files/file_000")`; partial selection by + filename prefix instead of an explicit path set. - `switch_one_forward/back`: switch to/from a view with one changed line. - `switch_many_forward/back`: switch to/from a view changing one line per file. @@ -67,6 +75,52 @@ favors spans. Database sizes are post-sample, uncompacted sizes, not a storage efficiency measurement. Errors and incomplete/duplicate samples fail the run; they are not passes. Fixtures are disposable; `--keep-fixtures` retains them. +## Git working-copy baseline + +`git_baseline.py` times Git doing the closest ordinary operation to a full +materialization, on a disposable Git repository built with the **same +synthetic content and edit count** as the atomic fixture (same file names, +line values, hot-file edit sequence, and one commit per edit): + +```sh +python3 -B tools/materialize-bench/git_baseline.py --profile many-files \ + --output /approved/scratch/git-01 +``` + +Cases: `checkout_full_noop` and `checkout_full_missing` (remove every +generated file, then `git checkout -- .`). Bytes, modes and a clean +`git status --porcelain` are verified after every sample outside timing. + +Boundary honesty for comparisons against a `materialize_bench` report from the +same machine and profile: + +- Git checkout restores from the **index**; it never touches pack files or + history. The Git analog of reading history is clone/fetch, which this + harness deliberately does not time. +- `total_ns` includes subprocess spawn overhead; atomic samples measure an + in-process API. Treat a few milliseconds as measurement overhead. +- Files Git considers unchanged are skipped by stat, so noop cases measure + Git's fast path, just as atomic's lease observation is its fast path. +- Reports are only comparable pairwise, same machine, same profile, close in + time; OS caches and system load remain uncontrolled. Ratios within one + report (partial against full) are more trustworthy than cross-tool ratios. + +## Agent-sandbox sync benchmark + +`run_agent_sandbox.py` builds and drives +`libatomic/examples/agent_sandbox_bench.rs`: a cold, empty sandbox syncing +over the production daemon `Materialize` RPC (tonic, Unix socket, +`SandboxImpl` handlers, `path_prefix` partial) versus Git's wire clone +(`file://`) on matched synthetic histories — the cold-sandbox scenario the +local cases above cannot answer. The benchmark checks expected bytes after +every sample; partial sandboxes must hold exactly the requested subtree and +nothing beyond it. + +```sh +python3 -B tools/materialize-bench/run_agent_sandbox.py --build \ + --profile many-files --output /approved/scratch/asb-01 +``` + The benchmark checks expected bytes and modes after every sample and verifies a single completed working-copy operation head. It does not certify imported history, genuine conflicts, cold caches, transport, watchers, crashes or power diff --git a/tools/materialize-bench/git_baseline.py b/tools/materialize-bench/git_baseline.py new file mode 100644 index 00000000..7334a262 --- /dev/null +++ b/tools/materialize-bench/git_baseline.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Timed Git working-copy baseline for materialization comparisons. + +Builds a disposable Git repository with the same synthetic history shapes as +atomic-repository/examples/materialize_bench.rs (same files, lines, edits and +commit count), then times `git checkout --` full working-tree restores after +removing files. Bytes and modes of every file, plus a clean `git status`, are +verified after every sample outside timing. + +This measures Git's worktree-write floor: checkout restores blob content from +the index. Git history reads (clone/fetch/pack access) are a different +operation and are not measured here. subprocess spawn overhead is inside +total_ns. OS caches are uncontrolled. Compare its report only against an +atomic materialize_bench run with the same profile on the same machine. +""" +import argparse +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import tempfile +import time + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from run import PROFILES, distribution # noqa: E402 + +CASES = ('checkout_full_noop', 'checkout_full_missing') + + +def generated_line(file_index, line_index): + return f'file_{file_index:04} line_{line_index:06} value_00000000\n' + + +def edited_line(line_index, edit): + return f'file_0000 line_{line_index:06} value_{edit + 1:08}\n' + + +def build_golden(files, lines): + golden = {} + for index in range(files): + golden[f'files/file_{index:04}.txt'] = ''.join( + generated_line(index, line) for line in range(lines)).encode() + golden['empty.txt'] = b'' + return golden + + +def git(root, *args, dates=None): + env = dict(os.environ) + env.update( + GIT_AUTHOR_NAME='bench', GIT_AUTHOR_EMAIL='bench@example.invalid', + GIT_COMMITTER_NAME='bench', GIT_COMMITTER_EMAIL='bench@example.invalid') + if dates is not None: + stamp = f'{1_600_000_000 + dates} +0000' + env.update(GIT_AUTHOR_DATE=stamp, GIT_COMMITTER_DATE=stamp) + result = subprocess.run(['git', *args], cwd=root, env=env, capture_output=True) + if result.returncode != 0: + raise RuntimeError(f'git {args[0]} failed: {result.stderr.decode(errors="replace")}') + return result.stdout + + +def build_fixture(root, files, lines, edits): + (root / 'files').mkdir(parents=True) + contents = {f'files/file_{index:04}.txt': + ''.join(generated_line(index, line) for line in range(lines)) + for index in range(files)} + for path, content in contents.items(): + (root / path).write_text(content) + (root / 'empty.txt').write_bytes(b'') + git(root, '-c', 'init.defaultBranch=dev', 'init', '-q') + git(root, 'add', '-A') + git(root, 'commit', '-q', '-m', 'synthetic baseline', dates=0) + hot = contents[f'files/file_0000.txt'].splitlines(keepends=True) + for edit in range(edits): + line = (edit * 17) % lines + hot[line] = edited_line(line, edit) + (root / 'files/file_0000.txt').write_text(''.join(hot)) + git(root, 'add', '-A') + git(root, 'commit', '-q', '-m', 'synthetic hot-file edit', dates=edit + 1) + # The golden state is the final dev tip, after every edit — the same + # capture point as the atomic fixture's "dev" golden. + golden = build_golden(files, lines) + golden['files/file_0000.txt'] = ''.join(hot).encode() + return golden + + +def verify(root, golden): + for path, content in golden.items(): + target = root / path + if not target.is_file(): + raise RuntimeError(f'missing after checkout: {path}') + if target.read_bytes() != content: + raise RuntimeError(f'bytes differ: {path}') + status = git(root, 'status', '--porcelain') + if status: + raise RuntimeError(f'working tree not clean: {status.decode(errors="replace")!r}') + + +def sample(root, golden, case): + if case == 'checkout_full_missing': + for path in sorted(golden): + (root / path).unlink() + started = time.monotonic_ns() + git(root, 'checkout', '--', '.') + total_ns = time.monotonic_ns() - started + # Filesystem equality and a clean index/status check are never timed. + verify(root, golden) + return {'case': case, 'total_ns': total_ns, 'phases_ns': {}, + 'verified_files': len(golden), + 'output_bytes': sum(len(content) for content in golden.values())} + + +def run(output, files, lines, edits, iterations, warmups, keep_fixtures): + if files <= 0 or lines < 2 or iterations < 1 or warmups < 0: + raise ValueError('invalid fixture or iteration configuration') + git_version = subprocess.run(['git', '--version'], capture_output=True, + check=True).stdout.decode().strip() + output.mkdir(parents=True, exist_ok=False) + output.chmod(0o700) + fixture = Path(tempfile.mkdtemp(dir=output, prefix='fixture-')) + metadata = {'outcome': 'incomplete', 'tool': 'git_baseline', 'schema': 1, + 'git_version': git_version, 'platform': platform.platform(), + 'logical_cpus': os.cpu_count(), + 'cache_policy': 'OS caches uncontrolled; not cold-cache claims', + 'boundary': 'git checkout restores from the index; pack/clone ' + 'history reads and subprocess spawn overhead are ' + 'inside total_ns'} + try: + golden = build_fixture(fixture, files, lines, edits) + samples = [] + for iteration in range(warmups + iterations): + for case in CASES: + measured = sample(fixture, golden, case) + if iteration >= warmups: + measured['iteration'] = iteration - warmups + samples.append(measured) + metadata.update(config=dict(files=files, lines=lines, edits=edits, + iterations=iterations, warmups=warmups), + fixture=dict(files=len(golden))) + metadata['samples'] = samples + (output / 'raw.json').write_text(json.dumps({'schema': 1, 'samples': samples}, indent=2) + '\n') + grouped = {} + for entry in samples: + grouped.setdefault(entry['case'], []).append(entry['total_ns'] / 1_000_000) + summary = {case: distribution(values) for case, values in grouped.items()} + metadata['outcome'] = 'measured' + metadata['summary_ms'] = summary + print('| Case | Median ms | Mean ms | p95 ms |') + print('|---|---:|---:|---:|') + for case in CASES: + row = summary[case] + print(f"| {case} | {row['median']:.3f} | {row['mean']:.3f} | {row['p95']:.3f} |") + print('Artifacts:', output) + except Exception as error: + metadata.update(outcome='failed-or-invalid', error=str(error)) + raise + finally: + (output / 'run.json').write_text(json.dumps(metadata, indent=2, sort_keys=True) + '\n') + if not keep_fixtures: + shutil.rmtree(fixture, ignore_errors=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True, type=Path, + help='new artifact directory; must not exist') + parser.add_argument('--profile', choices=PROFILES, default='many-files') + for name in ('files', 'lines', 'edits'): + parser.add_argument('--' + name, type=int) + parser.add_argument('--iterations', type=int, default=8) + parser.add_argument('--warmups', type=int, default=1) + parser.add_argument('--keep-fixtures', action='store_true') + args = parser.parse_args() + config = dict(PROFILES[args.profile]) + config.update({name: getattr(args, name) for name in ('files', 'lines', 'edits') + if getattr(args, name) is not None}) + run(args.output, iterations=args.iterations, warmups=args.warmups, + keep_fixtures=args.keep_fixtures, **config) + + +if __name__ == '__main__': + main() diff --git a/tools/materialize-bench/run.py b/tools/materialize-bench/run.py index 9f424c2b..6392a6a0 100644 --- a/tools/materialize-bench/run.py +++ b/tools/materialize-bench/run.py @@ -20,6 +20,7 @@ 'hot-file': dict(files=1, lines=4096, edits=48), } CASES = ('render_session', 'render_reopened', 'materialize_noop', 'materialize_missing', + 'materialize_paths_noop', 'materialize_paths_missing', 'materialize_prefix_missing', 'switch_one_forward', 'switch_one_back', 'switch_many_forward', 'switch_many_back') diff --git a/tools/materialize-bench/test_git_baseline.py b/tools/materialize-bench/test_git_baseline.py new file mode 100644 index 00000000..f6885e6c --- /dev/null +++ b/tools/materialize-bench/test_git_baseline.py @@ -0,0 +1,61 @@ +import importlib.util +import json +from pathlib import Path +import shutil +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('git_baseline', Path(__file__).with_name('git_baseline.py')) +git_baseline = importlib.util.module_from_spec(spec) +spec.loader.exec_module(git_baseline) + +GIT = shutil.which('git') is not None + + +@unittest.skipUnless(GIT, 'git executable required') +class GitBaselineTests(unittest.TestCase): + def test_generated_content_matches_atomic_fixture_format(self): + golden = git_baseline.build_golden(3, 8) + self.assertEqual( + golden['files/file_0002.txt'], + ''.join(git_baseline.generated_line(2, line) for line in range(8)).encode()) + self.assertEqual(git_baseline.edited_line(3, 1), + 'file_0000 line_000003 value_00000002\n') + + def test_fixture_history_and_clean_status(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / 'repo' + root.mkdir() + golden = git_baseline.build_fixture(root, 3, 8, 2) + self.assertEqual(golden['files/file_0000.txt'].count(b'value_00000001'), 1) + git_baseline.verify(root, golden) + + def test_smoke_run_measures_every_case_and_verifies(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / 'out' + git_baseline.run(output, files=3, lines=8, edits=2, + iterations=2, warmups=1, keep_fixtures=False) + raw = json.loads((output / 'raw.json').read_text()) + cases = {(s['case'], s['iteration']) for s in raw['samples']} + self.assertEqual(cases, {(case, i) for case in git_baseline.CASES + for i in range(2)}) + for entry in raw['samples']: + self.assertGreater(entry['total_ns'], 0) + run_report = json.loads((output / 'run.json').read_text()) + self.assertEqual(run_report['outcome'], 'measured') + self.assertEqual(len(run_report['summary_ms']), len(git_baseline.CASES)) + + def test_corrupted_fixture_fails_verification(self): + if not GIT: + self.skipTest('git executable required') + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / 'repo' + root.mkdir() + golden = git_baseline.build_fixture(root, 2, 4, 1) + (root / 'files/file_0000.txt').write_text('corrupted\n') + with self.assertRaises(RuntimeError): + git_baseline.verify(root, golden) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/materialize-bench/test_runner.py b/tools/materialize-bench/test_runner.py index abcf58d3..feeacbd2 100644 --- a/tools/materialize-bench/test_runner.py +++ b/tools/materialize-bench/test_runner.py @@ -20,7 +20,7 @@ def test_complete_report_and_summary(self): report = self.report() runner.validate_report(report, 3) summary = runner.summarize(report) - self.assertEqual(len(summary), 8) + self.assertEqual(len(summary), len(runner.CASES)) for row in summary: self.assertEqual(row['graph_over_span_median'], 2) self.assertEqual(row['graph']['n'], 3) From 827337a0f8217e4dfff3bad71244bd5a78f61376 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 9 Nov 2026 19:02:29 -0600 Subject: [PATCH 5/9] feat: agent-sandbox sync benchmark over the daemon Materialize RPC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add libatomic/examples/agent_sandbox_bench.rs plus a frozen runner: a cold, empty sandbox syncs over the production tonic stack (SandboxImpl handlers on a Unix socket, streaming Materialize with path_prefix, write_sandbox_entry receiver), with git clone over file:// as the wire-protocol comparison on the same synthetic history shape. Partial sandboxes must hold exactly the requested subtree and nothing beyond it. Recorded loopback runs: full-sync parity with git clone at both scales (66 vs 72 ms small, 190 vs 185 ms large). The atomic prefix partial holds a near-constant request phase because the handler walks the whole view closure and filters at stream time — filtering during projection is the measured next optimization. --- Cargo.lock | 3 + libatomic/Cargo.toml | 5 + libatomic/examples/agent_sandbox_bench.rs | 720 ++++++++++++++++++ tools/materialize-bench/run_agent_sandbox.py | 170 +++++ .../test_run_agent_sandbox.py | 51 ++ 5 files changed, 949 insertions(+) create mode 100644 libatomic/examples/agent_sandbox_bench.rs create mode 100644 tools/materialize-bench/run_agent_sandbox.py create mode 100644 tools/materialize-bench/test_run_agent_sandbox.py diff --git a/Cargo.lock b/Cargo.lock index 3e1e2d6d..16cdfc26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2358,6 +2358,7 @@ dependencies = [ "chrono", "data-encoding", "futures-core", + "hyper-util", "napi", "napi-build", "napi-derive", @@ -2366,6 +2367,7 @@ dependencies = [ "protox", "rand 0.8.8", "redb 4.4.0", + "serde", "serde_json", "similar", "tempfile", @@ -2373,6 +2375,7 @@ dependencies = [ "tokio-stream", "tonic", "tonic-build", + "tower 0.4.13", "tracing", "ulid", "url", diff --git a/libatomic/Cargo.toml b/libatomic/Cargo.toml index 0f26e962..a64ca7c8 100644 --- a/libatomic/Cargo.toml +++ b/libatomic/Cargo.toml @@ -57,6 +57,11 @@ url = { workspace = true } # Loopback protocol acceptance harness only; no production listener is added. axum = { version = "0.7", default-features = false, features = ["http1", "tokio"] } tempfile = { workspace = true } +serde = { workspace = true, features = ["derive"] } +# agent_sandbox_bench: the real tonic client stack over a Unix socket. +hyper-util = { workspace = true } +tower = { workspace = true } +tokio-stream = { workspace = true, features = ["net"] } [build-dependencies] napi-build = { version = "2", optional = true } diff --git a/libatomic/examples/agent_sandbox_bench.rs b/libatomic/examples/agent_sandbox_bench.rs new file mode 100644 index 00000000..5d2d139f --- /dev/null +++ b/libatomic/examples/agent_sandbox_bench.rs @@ -0,0 +1,720 @@ +//! Agent-sandbox sync benchmark: cold sandbox over the daemon RPC versus Git +//! over its wire protocol. +//! +//! The scenario: an agent sandbox starts with nothing and fetches the files +//! it needs. Three matched cases on the same synthetic history (two directory +//! subtrees plus an empty file, one commit per hot-file edit): +//! +//! - `atomic_rpc_full` — a fresh sandbox connects to a Unix-socket tonic +//! server hosting the production `SandboxImpl` handler and streams the +//! whole view via the `Materialize` RPC (no prefix), writing every entry. +//! - `atomic_rpc_partial` — the same `Materialize` RPC with `path_prefix` +//! set to one subtree; only that subtree's entries and their ancestor +//! directories may be streamed. +//! - `git_clone_full` — `git clone` of a bare repository over the `file://` +//! wire transport. +//! +//! The atomic side exercises the real transport end to end: a fresh channel +//! per sample (a sandbox never reuses a connection), the generated client, +//! the production handlers against a real repository, and the real +//! `write_sandbox_entry` receiver. Only the `atomicd` shell is absent; the +//! socket is bound directly. Only the sandbox token/middleware layer is +//! bypassed (no token is presented, so the handlers run as the trusted +//! local caller). +//! +//! Timing boundaries: the timed region starts at the client connect (or the +//! first Git subprocess spawn) and ends when the last entry is written (or +//! Git exits). Fixtures, server startup, and byte verification are outside +//! timing. There is no network RTT: the atomic transport is a loopback Unix +//! socket and Git uses loopback `file://` pack negotiation; treat absolute +//! numbers as transport-plus-work measurements, not WAN estimates. +use std::collections::BTreeMap; +use std::fs; +use std::io::Write; +use std::os::unix::net::UnixListener as StdUnixListener; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{bail, ensure, Context, Result}; +use libatomic::atomic::{sandbox_service_client::SandboxServiceClient, MaterializeRequest}; +use libatomic::daemon::services_sandbox::SandboxImpl; +use libatomic::daemon::state::DaemonState; +use serde::{Deserialize, Serialize}; +use tokio_stream::wrappers::UnixListenerStream; +use tonic::transport::{Endpoint, Uri}; +use tower::service_fn; +use hyper_util::rt::TokioIo; + +const CASES: &[&str] = &[ + "atomic_rpc_full", + "atomic_rpc_partial", + "git_clone_full", +]; + +/// The partial subtree every partial case fetches: `files/east/`. +const PARTIAL_PREFIX: &str = "files/east"; + +#[derive(Clone, Debug, Serialize, Deserialize)] +struct Config { + output: PathBuf, + files_east: usize, + files_west: usize, + lines: usize, + edits: usize, + iterations: usize, + warmups: usize, + allow_debug: bool, + keep_fixtures: bool, +} + +impl Config { + fn parse() -> Result { + let mut config = Self { + output: "agent-sandbox-bench.json".into(), + files_east: 8, + files_west: 16, + lines: 128, + edits: 24, + iterations: 8, + warmups: 1, + allow_debug: false, + keep_fixtures: false, + }; + let mut args = std::env::args().skip(1); + while let Some(arg) = args.next() { + match arg.as_str() { + "--allow-debug" => config.allow_debug = true, + "--keep-fixtures" => config.keep_fixtures = true, + "--output" => config.output = args.next().context("--output needs a path")?.into(), + "--files-east" => { + config.files_east = args.next().context("--files-east needs a value")?.parse()? + } + "--files-west" => { + config.files_west = args.next().context("--files-west needs a value")?.parse()? + } + "--lines" => config.lines = args.next().context("--lines needs a value")?.parse()?, + "--edits" => config.edits = args.next().context("--edits needs a value")?.parse()?, + "--iterations" => { + config.iterations = args.next().context("--iterations needs a value")?.parse()? + } + "--warmups" => { + config.warmups = args.next().context("--warmups needs a value")?.parse()? + } + _ => bail!("unknown argument: {arg}"), + } + } + config.validate()?; + Ok(config) + } + + fn validate(&self) -> Result<()> { + ensure!( + self.files_east > 0 + && self.files_west > 0 + && self.lines >= 2 + && self.iterations > 0, + "file counts and iterations must be positive; lines must be at least two" + ); + ensure!( + self.files_east + self.files_west <= 1024 + && self.lines <= 100_000 + && self.edits <= 10_000, + "fixture exceeds benchmark safety bounds" + ); + ensure!( + self.iterations <= 1000 + && self.warmups <= 1000 + && self + .files_east + .checked_add(self.files_west) + .is_some_and(|n| n.checked_mul(self.lines).is_some_and(|n| n <= 2_000_000)), + "sample count or total fixture lines exceed safety bounds" + ); + ensure!( + !cfg!(debug_assertions) || self.allow_debug, + "use a --release build (or --allow-debug for correctness smoke tests only)" + ); + ensure!( + !self.output.exists(), + "refusing to overwrite an existing output report" + ); + Ok(()) + } +} + +// ————————————————————————————— fixture ————————————————————————————— + +fn generated_line(tree: &str, file: usize, line: usize) -> String { + format!("{tree} file_{file:04} line_{line:06} value_00000000\n") +} + +fn edited_line(line: usize, edit: usize) -> String { + format!("east file_0000 line_{line:06} value_{edit:08}\n") +} + +/// One tracked file's golden bytes. +#[derive(Clone)] +struct Golden { + path: String, + content: Vec, +} + +fn build_golden(config: &Config) -> Vec { + let mut golden = Vec::new(); + for (tree, count) in [ + ("east", config.files_east), + ("west", config.files_west), + ] { + for file in 0..count { + golden.push(Golden { + path: format!("files/{tree}/file_{file:04}.txt"), + content: (0..config.lines) + .map(|line| generated_line(tree, file, line)) + .collect::() + .into_bytes(), + }); + } + } + golden.push(Golden { + path: "empty.txt".into(), + content: Vec::new(), + }); + golden +} + +fn east_paths(config: &Config) -> Vec { + (0..config.files_east) + .map(|file| format!("files/east/file_{file:04}.txt")) + .collect() +} + +fn git(root: &Path, args: &[&str], tick: Option) -> Result<()> { + let output = Command::new("git") + .args(args) + .current_dir(root) + .env("GIT_AUTHOR_NAME", "bench") + .env("GIT_AUTHOR_EMAIL", "bench@example.invalid") + .env("GIT_COMMITTER_NAME", "bench") + .env("GIT_COMMITTER_EMAIL", "bench@example.invalid") + .env( + "GIT_AUTHOR_DATE", + format!("{} +0000", 1_600_000_000 + tick.unwrap_or(0)), + ) + .env( + "GIT_COMMITTER_DATE", + format!("{} +0000", 1_600_000_000 + tick.unwrap_or(0)), + ) + .output() + .with_context(|| format!("running git {}", args.join(" ")))?; + ensure!( + output.status.success(), + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&output.stderr) + ); + Ok(()) +} + +/// Build the atomic host repository and the Git bare repository with the +/// same synthetic content. Returns the atomic golden and the bare path. +fn prepare_fixtures( + parent: &Path, + config: &Config, +) -> Result<(tempfile::TempDir, Vec, PathBuf)> { + let directory = tempfile::Builder::new() + .prefix("agb") + .tempdir_in(parent)?; + let root = directory.path(); + let golden = build_golden(config); + + // Atomic host repository. Writes happen through the ordinary record path + // so the served graph is a real repository, not a canned fixture. + let repo = atomic_repository::Repository::init(root.join("host"))?; + fs::create_dir_all(root.join("host/files/east"))?; + fs::create_dir_all(root.join("host/files/west"))?; + for file in &golden { + fs::write(root.join("host").join(&file.path), &file.content)?; + } + let wc = repo.require_working_copy_id()?; + repo.add(wc, "files", Default::default())?; + repo.add(wc, "empty.txt", Default::default())?; + record(&repo, "synthetic baseline", 0)?; + // Hot-file edits: every edit rewrites the east hot file and records it, + // exactly one commit per edit to match the Git history's shape. + let hot_path = root.join("host/files/east/file_0000.txt"); + let mut hot: Vec = (0..config.lines) + .map(|l| generated_line("east", 0, l)) + .collect(); + for edit in 0..config.edits { + let line = (edit * 17) % config.lines; + hot[line] = edited_line(line, edit + 1); + let content = hot.concat(); + fs::write(&hot_path, &content)?; + record(&repo, "synthetic hot-file edit", edit + 1)?; + } + let hot = hot.concat(); + // The golden east hot file must reflect the final edit, not the baseline. + let mut golden = golden; + let golden_east_hot = golden + .iter_mut() + .find(|f| f.path == "files/east/file_0000.txt") + .context("missing east hot file in golden")?; + golden_east_hot.content = hot.into_bytes(); + drop(repo); + + // Git bare repository with the same shape: one baseline commit, one + // commit per hot-file edit. The seed must start from the baseline + // content — the golden's hot file already reflects the edits. + let seed = root.join("git-seed"); + fs::create_dir_all(seed.join("files/east"))?; + fs::create_dir_all(seed.join("files/west"))?; + let baseline_hot: Vec = golden + .iter() + .map(|file| { + if file.path == "files/east/file_0000.txt" { + Golden { + path: file.path.clone(), + content: (0..config.lines) + .map(|line| generated_line("east", 0, line)) + .collect::() + .into_bytes(), + } + } else { + file.clone() + } + }) + .collect(); + for file in &baseline_hot { + fs::write(seed.join(&file.path), &file.content)?; + } + git(&seed, &["init", "-q", "-b", "dev"], None)?; + git(&seed, &["add", "-A"], None)?; + git(&seed, &["commit", "-q", "-m", "synthetic baseline"], Some(0))?; + let seed_hot = seed.join("files/east/file_0000.txt"); + for edit in 0..config.edits { + let line = (edit * 17) % config.lines; + let mut lines: Vec = fs::read_to_string(&seed_hot)? + .split_inclusive('\n') + .map(String::from) + .collect(); + lines[line] = edited_line(line, edit + 1); + fs::write(&seed_hot, lines.concat())?; + git(&seed, &["add", "-A"], None)?; + git(&seed, &["commit", "-q", "-m", "synthetic hot-file edit"], Some(edit + 1))?; + } + let bare = root.join("host-bare.git"); + git( + root, + &["clone", "-q", "--bare", "git-seed", "host-bare.git"], + None, + )?; + // Both histories must have replayed to byte-identical trees. + let git_hot = fs::read(&seed_hot)?; + ensure!( + golden + .iter() + .find(|f| f.path == "files/east/file_0000.txt") + .is_some_and(|f| f.content == git_hot), + "atomic and Git fixture hot files diverged" + ); + Ok((directory, golden, bare)) +} + +fn record(repo: &atomic_repository::Repository, message: &str, tick: usize) -> Result<()> { + use atomic_core::change::ChangeHeader; + use atomic_repository::RecordOptions; + let mut header = ChangeHeader::new(message); + header.timestamp = chrono::DateTime::from_timestamp(1_600_000_000 + tick as i64, 0).unwrap(); + repo.record( + repo.require_working_copy_id()?, + header, + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true) + .sync_vault(false) + .enrich_kg(false), + )?; + Ok(()) +} + +// ————————————————————————————— server ————————————————————————————— + +/// Bind the production sandbox handlers on a private Unix socket. This is +/// the `atomicd` shape minus the shell: one `DaemonState`, one registered +/// repository, the generated tonic server. +fn serve( + state: Arc, + host_root: PathBuf, + socket: PathBuf, +) -> Result>> { + use libatomic::atomic::sandbox_service_server::SandboxServiceServer; + + let std_listener = StdUnixListener::bind(&socket) + .with_context(|| format!("binding {}", socket.display()))?; + std_listener.set_nonblocking(true)?; + // Fail fast if the repository cannot open; the server would otherwise + // serve errors that look like client bugs. The registration also gives + // the caller the repository ref the handlers will resolve. + let handle = state.register(host_root); + handle.repository()?; + Ok(std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|e| anyhow::anyhow!("server runtime: {e}"))?; + runtime.block_on(async move { + let listener = tokio::net::UnixListener::from_std(std_listener)?; + let incoming = UnixListenerStream::new(listener); + tonic::transport::Server::builder() + .add_service(SandboxServiceServer::new(SandboxImpl { state })) + .serve_with_incoming(incoming) + .await + .map_err(|e| anyhow::anyhow!("tonic server: {e}")) + }) + })) +} + +// ————————————————————————————— client ————————————————————————————— + +async fn connect(socket: &Path) -> Result> { + let socket = socket.to_path_buf(); + let channel = Endpoint::from_static("http://localhost") + .connect_with_connector(service_fn(move |_: Uri| { + let socket = socket.clone(); + async move { + let stream = tokio::net::UnixStream::connect(socket).await?; + Ok::<_, std::io::Error>(TokioIo::new(stream)) + } + })) + .await?; + Ok(SandboxServiceClient::new(channel).max_decoding_message_size(64 * 1024 * 1024)) +} + +/// One cold sandbox materialize: fresh channel, `Materialize` RPC, write +/// every streamed entry. Connect, request and stream run on one runtime — +/// a tonic channel is bound to the runtime that created it. +fn atomic_sandbox( + socket: &Path, + repository: libatomic::atomic::RepositoryRef, + target: libatomic::atomic::ViewRef, + prefix: Option, + sandbox: &Path, +) -> Result<(BTreeMap, usize)> { + use libatomic::daemon::sandbox_wire; + + let mut phases = BTreeMap::new(); + let sandbox = sandbox.to_path_buf(); + let socket = socket.to_path_buf(); + let (written, inner) = std::thread::spawn(move || -> Result<(usize, BTreeMap)> { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(anyhow::Error::msg)?; + runtime.block_on(async move { + use tokio_stream::StreamExt; + let mut phases = BTreeMap::new(); + let started = Instant::now(); + let mut client = connect(&socket).await?; + phases.insert("connect".into(), started.elapsed().as_nanos()); + let started = Instant::now(); + let mut stream = client + .materialize(MaterializeRequest { + repository: Some(repository), + target: Some(target), + path_prefix: prefix, + }) + .await? + .into_inner(); + phases.insert("request".into(), started.elapsed().as_nanos()); + let started = Instant::now(); + let mut written = 0usize; + loop { + let frame = stream + .next() + .await + .ok_or_else(|| anyhow::anyhow!("materialize ended without its summary"))? + .map_err(anyhow::Error::msg)?; + match frame.frame { + Some(libatomic::atomic::materialize_frame::Frame::Entry(entry)) => { + atomic_repository::write_sandbox_entry( + &sandbox, + &entry.path, + sandbox_wire::tree_entry_kind(&entry).map_err(anyhow::Error::msg)?, + entry.mode.unwrap_or(0o644), + entry.inline.as_deref().unwrap_or_default(), + )?; + written += 1; + } + Some(libatomic::atomic::materialize_frame::Frame::Done(done)) => { + ensure!( + done.entries as usize == written, + "the server sent {written} entries but said {}", + done.entries + ); + break; + } + None => bail!("an empty materialize frame"), + } + } + phases.insert("stream_and_write".into(), started.elapsed().as_nanos()); + Ok((written, phases)) + }) + }) + .join() + .map_err(|_| anyhow::anyhow!("client thread panicked"))??; + phases.extend(inner); + Ok((phases, written)) +} + +// ————————————————————————————— git cases ————————————————————————————— + +fn git_clone_full(bare: &Path, sandbox: &Path) -> Result> { + let mut phases = BTreeMap::new(); + let started = Instant::now(); + git( + sandbox.parent().unwrap_or(Path::new("/")), + &[ + "clone", + "-q", + &format!("file://{}", bare.display()), + &sandbox.display().to_string(), + ], + None, + )?; + phases.insert("clone".into(), started.elapsed().as_nanos()); + Ok(phases) +} + +// ————————————————————————————— verification ————————————————————————————— + +fn file_mode(path: &Path) -> Result { + use std::os::unix::fs::PermissionsExt; + Ok(fs::metadata(path)?.permissions().mode() & 0o7777) +} + +/// Full sandboxes hold every tracked file; partial sandboxes hold exactly +/// the east subtree and nothing else from the west or the root. +fn verify(sandbox: &Path, golden: &[Golden], partial: bool, east: &[String]) -> Result<()> { + if partial { + for path in east { + let content = golden + .iter() + .find(|f| &f.path == path) + .context("east path missing from golden")?; + let target = sandbox.join(path); + ensure!(target.is_file(), "partial sandbox misses {}", path); + ensure!( + fs::read(&target)? == content.content, + "partial sandbox bytes differ: {path}" + ); + } + for file in golden { + if !east.contains(&file.path) { + ensure!( + !sandbox.join(&file.path).exists(), + "partial sandbox fetched beyond its prefix: {}", + file.path + ); + } + } + return Ok(()); + } + for file in golden { + let target = sandbox.join(&file.path); + ensure!(target.is_file(), "sandbox misses {}", file.path); + ensure!( + fs::read(&target)? == file.content, + "sandbox bytes differ: {}", + file.path + ); + ensure!( + file_mode(&target)? == 0o644 || file.content.is_empty(), + "sandbox mode differs: {}", + file.path + ); + } + Ok(()) +} + +// ————————————————————————————— harness ————————————————————————————— + +#[derive(Debug, Serialize)] +struct Sample { + case: String, + iteration: usize, + total_ns: u128, + phases_ns: BTreeMap, + entries: Option, + verified_files: usize, + output_bytes: usize, +} + +fn run(config: Config) -> Result<()> { + config.validate()?; + let parent = config + .output + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + fs::create_dir_all(parent)?; + ensure!( + Command::new("git") + .arg("--version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false), + "git is required for the Git comparison cases" + ); + let (fixtures, golden, bare) = prepare_fixtures(parent, &config)?; + let host_root = fixtures.path().join("host"); + let socket = fixtures.path().join("s.sock"); + ensure!( + socket.to_string_lossy().len() <= 100, // sun_path limit is 104; leave slack + "socket path too long for a Unix socket: {} — use a shorter --output directory", + socket.display() + ); + let state = Arc::new(DaemonState::new()); + let handle = state.register(host_root.clone()); + let repository = handle.repository_ref().repository_id; + handle.repository()?; // fail fast; the server would serve errors otherwise + let server = serve(state, host_root, socket.clone())?; + let east = east_paths(&config); + let east_bytes: usize = golden + .iter() + .filter(|f| east.contains(&f.path)) + .map(|f| f.content.len()) + .sum(); + + let mut samples = Vec::new(); + let result = (|| -> Result<()> { + for iteration in 0..config.warmups + config.iterations { + for case in CASES { + let partial = case.contains("partial"); + let sandbox = fixtures.path().join(format!("sandbox-{case}-{iteration}")); + let started = Instant::now(); + let (phases, entries) = if case.starts_with("atomic") { + let target = libatomic::atomic::ViewRef { + view_id: Vec::new(), + name: Some("dev".into()), + }; + let reference = libatomic::atomic::RepositoryRef { + authority: "local".into(), + repository_id: repository.clone(), + workspace_id: None, + }; + let (phases, entries) = atomic_sandbox( + &socket, + reference, + target, + partial.then(|| PARTIAL_PREFIX.to_string()), + &sandbox, + )?; + (phases, Some(entries)) + } else if *case == "git_clone_full" { + (git_clone_full(&bare, &sandbox)?, None) + } else { + bail!("unknown case: {case}") + }; + let total_ns = started.elapsed().as_nanos(); + // Byte verification is never timed. + verify(&sandbox, &golden, partial, &east) + .with_context(|| format!("{case} iteration {iteration} verification"))?; + if iteration >= config.warmups { + samples.push(Sample { + case: case.to_string(), + iteration: iteration - config.warmups, + total_ns, + phases_ns: phases, + entries, + verified_files: if partial { east.len() } else { golden.len() }, + output_bytes: if partial { + east_bytes + } else { + golden.iter().map(|f| f.content.len()).sum() + }, + }); + } + let _ = fs::remove_dir_all(&sandbox); + } + } + Ok(()) + })(); + // The tonic server never shuts down; detach it. The process (or test) + // exits with the thread still serving, which mirrors atomicd. + drop(server); + if let Err(error) = result { + // Retain the fixtures for inspection; a TempDir would delete them. + let retained = fixtures.keep(); + eprintln!("fixtures retained at {} for inspection", retained.display()); + return Err(error); + } + let retained_fixtures = config.keep_fixtures.then(|| fixtures.keep()); + + let report = serde_json::json!({ + "schema": 1, + "config": config, + "debug_build": cfg!(debug_assertions), + "platform": {"os": std::env::consts::OS, "arch": std::env::consts::ARCH}, + "fixture_kind": "two-subtree synthetic histories; atomic daemon Materialize RPC on a Unix socket versus Git file:// clone", + "cache_policy": "server and bare repository persist across samples; sandboxes are fresh; OS caches uncontrolled", + "retained_fixtures": retained_fixtures, + "samples": samples, + }); + let mut output = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&config.output)?; + serde_json::to_writer_pretty(&mut output, &report)?; + output.flush()?; + println!( + "verified {} samples; report {}", + config.iterations * CASES.len(), + config.output.display() + ); + Ok(()) +} + +fn main() -> Result<()> { + run(Config::parse()?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn smoke_checks_every_case() { + let dir = tempfile::tempdir().unwrap(); + let output = dir.path().join("report.json"); + run(Config { + output: output.clone(), + files_east: 2, + files_west: 2, + lines: 8, + edits: 1, + iterations: 1, + warmups: 0, + allow_debug: true, + keep_fixtures: false, + }) + .unwrap(); + let report: serde_json::Value = serde_json::from_slice(&fs::read(output).unwrap()).unwrap(); + let samples = report["samples"].as_array().unwrap(); + assert_eq!(samples.len(), CASES.len()); + for sample in samples { + assert!(sample["total_ns"].as_u64().unwrap() > 0); + assert!(sample["verified_files"].as_u64().unwrap() > 0); + } + let partial: Vec<_> = samples + .iter() + .filter(|s| s["case"].as_str().unwrap().contains("partial")) + .collect(); + assert_eq!(partial.len(), 1); + assert_eq!(partial[0]["case"].as_str().unwrap(), "atomic_rpc_partial"); + assert_eq!(partial[0]["verified_files"].as_u64().unwrap(), 2); + } +} diff --git a/tools/materialize-bench/run_agent_sandbox.py b/tools/materialize-bench/run_agent_sandbox.py new file mode 100644 index 00000000..29e9491f --- /dev/null +++ b/tools/materialize-bench/run_agent_sandbox.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Frozen, local-only agent-sandbox sync runs: atomic daemon RPC versus Git wire clone. + +Wraps libatomic/examples/agent_sandbox_bench.rs with the same identity +discipline as run.py: the source tree is fingerprinted before and after build +and measurement, the release executable is copied into the artifact directory +and its SHA-256 is verified afterwards. Debug runs are smoke tests, never +performance evidence. +""" +import argparse +import json +from collections import defaultdict +import os +from pathlib import Path +import platform +import shutil +import subprocess +import time + +sys_path = Path(__file__).resolve().parent +import sys # noqa: E402 + +sys.path.insert(0, str(sys_path)) +from run import digest, distribution, source_fingerprint # noqa: E402 + +ROOT = sys_path.parents[1] +PROFILES = { + 'smoke': dict(files_east=2, files_west=2, lines=8, edits=1), + 'many-files': dict(files_east=8, files_west=16, lines=128, edits=24), + 'hot-file': dict(files_east=1, files_west=16, lines=4096, edits=48), +} +CASES = ('atomic_rpc_full', 'atomic_rpc_partial', 'git_clone_full') + + +def validate_report(report, iterations): + expected = {(case, iteration) for case in CASES for iteration in range(iterations)} + seen = set() + for sample in report['samples']: + key = (sample['case'], sample['iteration']) + if key not in expected or key in seen: + raise ValueError('unexpected or duplicate sample: ' + repr(key)) + seen.add(key) + if not isinstance(sample['total_ns'], int) or sample['total_ns'] <= 0: + raise ValueError('nonpositive or invalid sample duration') + if seen != expected: + raise ValueError('not all cases produced samples') + + +def summarize(report): + groups = defaultdict(list) + phases = defaultdict(lambda: defaultdict(list)) + for sample in report['samples']: + groups[sample['case']].append(sample['total_ns'] / 1_000_000) + for name, nanos in sample['phases_ns'].items(): + phases[sample['case']][name].append(nanos / 1_000_000) + return {case: dict(distribution(values), + phases_ms={name: distribution(values) for name, values in phases[case].items()}) + for case, values in groups.items()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument('--build', action='store_true', + help='build offline/locked/release from fingerprinted source') + selection.add_argument('--binary', type=Path, + help='existing agent_sandbox_bench executable (build provenance not assumed)') + parser.add_argument('--output', required=True, type=Path, + help='new artifact directory; must not exist') + parser.add_argument('--profile', choices=PROFILES, default='many-files') + for name in ('files-east', 'files-west', 'lines', 'edits'): + parser.add_argument('--' + name, type=int) + parser.add_argument('--iterations', type=int, default=8) + parser.add_argument('--warmups', type=int, default=1) + parser.add_argument('--keep-fixtures', action='store_true') + parser.add_argument('--allow-debug', action='store_true', + help='correctness smoke only, not performance evidence') + args = parser.parse_args() + if args.iterations < 1 or args.warmups < 0: + parser.error('iterations must be positive and warmups nonnegative') + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + output.chmod(0o700) + source_before = source_fingerprint() + metadata = dict(outcome='incomplete', source_sha256=source_before, + source_binding='fingerprinted offline release build' if args.build + else 'caller-supplied binary; source/build association unverified', + platform=platform.platform(), logical_cpus=os.cpu_count(), + concurrency_control='OS cache and unrelated system load are not controlled') + try: + candidate = args.binary.resolve() if args.binary else None + if args.build: + command = ['cargo', 'build', '--release', '--offline', '--locked', '-p', 'libatomic', + '--example', 'agent_sandbox_bench', '--message-format=json-render-diagnostics'] + build_env = dict(os.environ, CARGO_NET_OFFLINE='true') + build_env.setdefault('CARGO_BUILD_JOBS', '2') + build_env.setdefault('CARGO_PROFILE_RELEASE_STRIP', 'none') + metadata['build_command'] = command + result = subprocess.run(command, cwd=ROOT, env=build_env, capture_output=True) + (output / 'build.stdout.jsonl').write_bytes(result.stdout) + (output / 'build.stderr.log').write_bytes(result.stderr) + result.check_returncode() + for line in result.stdout.splitlines(): + event = json.loads(line) + if event.get('reason') == 'compiler-artifact' and \ + event.get('target', {}).get('name') == 'agent_sandbox_bench' and \ + event.get('executable'): + candidate = Path(event['executable']) + if candidate is None: + raise RuntimeError('Cargo did not report the benchmark executable') + if source_fingerprint() != source_before: + raise RuntimeError('source changed during build; discard this candidate') + if candidate is None or not candidate.is_file(): + raise RuntimeError('benchmark executable not found') + binary = output / 'agent_sandbox_bench' + shutil.copy2(candidate, binary) + metadata['binary_sha256'] = digest(binary) + config = {name.replace('_', '-'): value + for name, value in PROFILES[args.profile].items()} + for name in ('files-east', 'files-west', 'lines', 'edits'): + value = getattr(args, name.replace('-', '_')) + if value is not None: + config[name] = value + command = [str(binary), '--output', str(output / 'raw.json'), + '--iterations', str(args.iterations), '--warmups', str(args.warmups)] + for name, value in config.items(): + command += ['--' + name, str(value)] + if args.keep_fixtures: + command.append('--keep-fixtures') + if args.allow_debug: + command.append('--allow-debug') + home = output / 'home' + temporary = output / 'tmp' + home.mkdir() + temporary.mkdir() + env = {key: value for key, value in os.environ.items() + if not key.startswith(('ATOMIC_', 'GIT_', 'XDG_'))} + env.update(HOME=str(home), ATOMIC_HOME=str(home / '.atomic'), + ATOMIC_CONFIG_DIR=str(home / 'config'), ATOMIC_SERVICE='local', + TMPDIR=str(temporary)) + metadata['command'] = command + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, capture_output=True) + metadata['runner_elapsed_seconds'] = time.monotonic() - started + metadata['exit_code'] = result.returncode + (output / 'stdout.log').write_bytes(result.stdout) + (output / 'stderr.log').write_bytes(result.stderr) + result.check_returncode() + if source_fingerprint() != source_before or digest(binary) != metadata['binary_sha256']: + raise RuntimeError('source or pinned binary changed during measurement; results invalid') + report = json.loads((output / 'raw.json').read_text()) + validate_report(report, args.iterations) + summary = summarize(report) + metadata.update(outcome='smoke-only' if report['debug_build'] else 'measured', + summary_ms=summary) + print('| Case | Median ms | Mean ms | p95 ms |') + print('|---|---:|---:|---:|') + for case in CASES: + row = summary[case] + print(f"| {case} | {row['median']:.3f} | {row['mean']:.3f} | {row['p95']:.3f} |") + print('Artifacts:', output) + except Exception as error: + metadata.update(outcome='failed-or-invalid', error=str(error)) + raise + finally: + (output / 'run.json').write_text(json.dumps(metadata, indent=2, sort_keys=True) + '\n') + + +if __name__ == '__main__': + main() diff --git a/tools/materialize-bench/test_run_agent_sandbox.py b/tools/materialize-bench/test_run_agent_sandbox.py new file mode 100644 index 00000000..aebac217 --- /dev/null +++ b/tools/materialize-bench/test_run_agent_sandbox.py @@ -0,0 +1,51 @@ +import importlib.util +import json +from pathlib import Path +import unittest + +for name in ('run', 'run_agent_sandbox'): + spec = importlib.util.spec_from_file_location(name, Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + globals()[name] = module + + +def report(): + return {'samples': [dict(case=case, iteration=i, total_ns=(2 if case.startswith('atomic') else 1) * 1_000_000, + phases_ns={'connect': 100_000}) + for case in run_agent_sandbox.CASES for i in range(3)]} + + +class RunnerTests(unittest.TestCase): + def test_complete_report_and_summary(self): + data = report() + run_agent_sandbox.validate_report(data, 3) + summary = run_agent_sandbox.summarize(data) + self.assertEqual(len(summary), len(run_agent_sandbox.CASES)) + self.assertEqual(summary['atomic_rpc_full']['median'], 2) + self.assertEqual(summary['git_clone_full']['median'], 1) + self.assertEqual(summary['atomic_rpc_full']['phases_ms']['connect']['median'], .1) + + def test_missing_duplicate_unknown_and_invalid_samples_fail(self): + import copy + bad = copy.deepcopy(report()) + bad['samples'].pop() + with self.assertRaises(ValueError): + run_agent_sandbox.validate_report(bad, 3) + for field, value in [('case', 'unknown'), ('iteration', 9), ('total_ns', 0)]: + bad = copy.deepcopy(report()) + bad['samples'][0][field] = value + with self.assertRaises(ValueError): + run_agent_sandbox.validate_report(bad, 3) + bad = copy.deepcopy(report()) + bad['samples'][-1] = bad['samples'][0] + with self.assertRaises(ValueError): + run_agent_sandbox.validate_report(bad, 3) + + def test_profiles_shape_matches_example_flags(self): + for profile in run_agent_sandbox.PROFILES.values(): + self.assertEqual(set(profile), {'files_east', 'files_west', 'lines', 'edits'}) + + +if __name__ == '__main__': + unittest.main() From e17a0861ebebcb5777482e4736dba1c5ac1c4c66 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 9 Nov 2026 23:20:36 -0600 Subject: [PATCH 6/9] perf: cache verified change parses in the change store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit load_verified_change re-read and re-parsed the change file on every call. Change files are content-addressed and write-once, so a parsed, hash-verified change is immutable for the handle's lifetime; cache it. revalidate_verified_change callers keep the old re-read-and-compare behavior, and explicit revalidation requests bypass the cache. Measured on the agent-sandbox benchmark (2,048-file span fixture, 49 changes, one 11 MB baseline): a full view render went from 34.6 s to 1.08 s — every per-file span context was re-parsing the baseline, at roughly 85 ms per render. Span partial materialize dropped from 1.13 s to 634 ms. Graph-origin paths do not consult this cache and are unchanged. --- atomic-repository/src/changestore/mod.rs | 20 ++++++++++++++++++++ libatomic/examples/agent_sandbox_bench.rs | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/atomic-repository/src/changestore/mod.rs b/atomic-repository/src/changestore/mod.rs index b3d412d0..a546053f 100644 --- a/atomic-repository/src/changestore/mod.rs +++ b/atomic-repository/src/changestore/mod.rs @@ -122,6 +122,12 @@ pub struct ChangeStore { /// concurrent read access while ensuring exclusive write access. pub(crate) cache: RwLock>, + /// Parsed, hash-verified changes keyed by hash. Change files are + /// content-addressed and write-once, so a verified parse is immutable + /// for this handle's lifetime; re-parsing an 11 MB baseline per rendered + /// file is the dominant cost of span rendering. + pub(crate) verified_cache: RwLock>, + /// Content spans held without their change files — a remote sandbox's /// cache has the bytes of the vertices it was sent, never whole changes. /// Consulted before the change files. @@ -215,6 +221,7 @@ impl ChangeStore { Ok(Self { changes_dir, cache: RwLock::new(LruCache::new(cache_capacity)), + verified_cache: RwLock::new(LruCache::new(cache_capacity)), spans: RwLock::new(SpanOverlay::default()), }) } @@ -230,6 +237,7 @@ impl ChangeStore { Ok(Self { changes_dir, cache: RwLock::new(LruCache::new(cache_capacity)), + verified_cache: RwLock::new(LruCache::new(cache_capacity)), spans: RwLock::new(SpanOverlay::default()), }) } @@ -453,6 +461,13 @@ impl ChangeStore { hash: &Hash, cached: Option<&VerifiedChange>, ) -> ChangeStoreResult { + if cached.is_none() { + if let Ok(cache) = self.verified_cache.read() { + if let Some(verified) = cache.peek(hash) { + return Ok(verified.clone()); + } + } + } let bytes = fs::read(self.change_path(hash)).map_err(|error| { if error.kind() == std::io::ErrorKind::NotFound { ChangeStoreError::NotFound { @@ -472,6 +487,11 @@ impl ChangeStore { computed: verified.hash().to_base32(), }); } + if cached.is_none() { + if let Ok(mut cache) = self.verified_cache.write() { + cache.insert(*hash, verified.clone()); + } + } Ok(verified) } diff --git a/libatomic/examples/agent_sandbox_bench.rs b/libatomic/examples/agent_sandbox_bench.rs index 5d2d139f..f74142c5 100644 --- a/libatomic/examples/agent_sandbox_bench.rs +++ b/libatomic/examples/agent_sandbox_bench.rs @@ -67,6 +67,7 @@ struct Config { warmups: usize, allow_debug: bool, keep_fixtures: bool, + span: bool, } impl Config { @@ -81,11 +82,13 @@ impl Config { warmups: 1, allow_debug: false, keep_fixtures: false, + span: false, }; let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { match arg.as_str() { "--allow-debug" => config.allow_debug = true, + "--span" => config.span = true, "--keep-fixtures" => config.keep_fixtures = true, "--output" => config.output = args.next().context("--output needs a path")?.into(), "--files-east" => { @@ -232,6 +235,12 @@ fn prepare_fixtures( // Atomic host repository. Writes happen through the ordinary record path // so the served graph is a real repository, not a canned fixture. let repo = atomic_repository::Repository::init(root.join("host"))?; + if config.span { + use atomic_core::pristine::{CapabilityMutTxnT, MutTxnT, SPAN_RANGE_CAPABILITY}; + let mut txn = repo.pristine().write_txn()?; + txn.put_required_capability(SPAN_RANGE_CAPABILITY)?; + txn.commit()?; + } fs::create_dir_all(root.join("host/files/east"))?; fs::create_dir_all(root.join("host/files/west"))?; for file in &golden { From 66c2f6d7298b593735cff3db4b7e543448453319 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Mon, 9 Nov 2026 23:38:27 -0600 Subject: [PATCH 7/9] perf: share one span render context per view walk materialize_view_entries rendered each file through a fresh SpanRenderContext, and every context decodes and verifies the whole visible closure: a 2,048-file span view built 4,213 contexts per materialize (traced), ~20.7 s of combined CPU hidden by rayon parallelism. SharedSpanContext builds once per walk, lazily on the first span-rooted file, and is safe to share across the rayon render chunks (OnceLock; single-threaded OnceCell cannot cross them). Graph-origin files still fall back to the graph renderer, and callers without a shared context keep the old per-call path. Measured with the parse cache on the agent-sandbox benchmark (2,048-file span fixture): partial materialize 639 -> 341 ms, full 1,091 -> 794 ms. Span render now matches the graph backend on this fixture. --- atomic-repository/src/repository/content.rs | 76 +++++++++++++++++++ .../src/repository/materialize.rs | 27 ++++++- atomic-repository/src/repository/record.rs | 3 +- atomic-repository/src/repository/status.rs | 3 +- libatomic/examples/agent_sandbox_bench.rs | 1 + 5 files changed, 105 insertions(+), 5 deletions(-) diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index fa03ff60..60ab2238 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -928,6 +928,82 @@ where .map_err(|error| RepositoryError::Database(error.to_string())) } +/// One span render context shared by every file read through a single +/// transaction and visibility closure. `retrieve_span_if_present` builds a +/// fresh context per call — which decodes and verifies the whole visible +/// closure — so a per-file caller on a large view pays the closure decode +/// thousands of times; holders of this struct pay it once. Safe to share +/// across rayon workers: first use builds, the rest reuse. +pub(super) struct SharedSpanContext<'a, T, C> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + txn: &'a T, + store: &'a C, + visibility: &'a GraphVisibilityClosure, + context: std::sync::OnceLock< + Option>, + >, +} + +impl<'a, T, C> SharedSpanContext<'a, T, C> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT + + Sync, + C: atomic_core::change::ChangeStore + Sync, +{ + pub(super) fn new( + txn: &'a T, + store: &'a C, + visibility: &'a GraphVisibilityClosure, + ) -> Self { + Self { + txn, + store, + visibility, + context: std::sync::OnceLock::new(), + } + } + + /// Render `inode` through the shared context when its span root is + /// visible; `None` means the file is graph-origin and must be read from + /// the graph. + pub(super) fn render_if_present( + &self, + inode: Inode, + ) -> Result>, RepositoryError> { + if !has_visible_span_root(self.txn, self.store, inode, self.visibility)? { + return Ok(None); + } + let context = self + .context + .get_or_init(|| { + atomic_core::pristine::span_patch_storage::SpanRenderContext::new( + self.txn, + self.store, + self.visibility, + ) + .ok() + }) + .as_ref(); + match context { + Some(context) => context + .render(inode) + .map(Some) + .map_err(|error| RepositoryError::Database(error.to_string())), + // The closure could not be decoded this once; without a span + // context this file must be read from the graph like any other + // graph-origin path. + None => Ok(None), + } + } +} + pub(super) fn retrieve_span_bounded_if_present( txn: &T, store: &C, diff --git a/atomic-repository/src/repository/materialize.rs b/atomic-repository/src/repository/materialize.rs index 9fb37aa9..034d697b 100644 --- a/atomic-repository/src/repository/materialize.rs +++ b/atomic-repository/src/repository/materialize.rs @@ -435,6 +435,7 @@ pub(super) fn render_name_conflict( inode_graph_table, visibility, external_hashes, + None, path, *inode, *position, @@ -455,13 +456,25 @@ pub(super) fn render_name_conflict_side( inode_graph_table: &redb::ReadOnlyMultimapTable<&'static [u8; 32], &'static [u8; 24]>, visibility: &GraphVisibilityClosure, external_hashes: &std::collections::HashMap, + shared_spans: Option< + &super::content::SharedSpanContext< + '_, + atomic_core::pristine::ReadTxn, + C, + >, + >, path: &str, inode: Inode, position: Position, ) -> Result, String> { - if let Some(bytes) = super::content::retrieve_span_if_present(txn, store, inode, visibility) - .map_err(|error| format!("{path}: span: {error}"))? - { + let span_bytes = match shared_spans { + Some(shared) => shared + .render_if_present(inode) + .map_err(|error| format!("{path}: span: {error}"))?, + None => super::content::retrieve_span_if_present(txn, store, inode, visibility) + .map_err(|error| format!("{path}: span: {error}"))?, + }; + if let Some(bytes) = span_bytes { return Ok(bytes); } use atomic_core::output::repo::{ @@ -597,6 +610,7 @@ fn prepare_name_conflict_output( inode_graph_table, visibility, external_hashes, + None, &path, side.inode, side.position, @@ -755,6 +769,7 @@ impl Repository { &inode_graph_table, &visibility, &external_hashes, + None, path, item.inode, item.position, @@ -2589,6 +2604,11 @@ impl Repository { // Preserve the streaming client's mkdir-before-file contract. paths.sort_by(|a, b| is_directory(b).cmp(&is_directory(a)).then_with(|| a.cmp(b))); let filter = crate::content_filter::GitAttributesFilter::for_repository(&self.root); + // One span context per walk: per-file contexts each decode and + // verify the whole visible closure, which on a large span-backed + // view costs more than the render itself. + let shared_spans = + super::content::SharedSpanContext::new(&txn, &self.change_store, &visibility); for chunk in paths.chunks(256) { let rendered: Vec> = chunk .par_iter() @@ -2625,6 +2645,7 @@ impl Repository { &table, &visibility, &hashes, + Some(&shared_spans), path, item.inode, item.position, diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index c4e13d0b..d255d9b1 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -5075,7 +5075,8 @@ fn prepare_name_resolution( &inode_graph_table, visibility, &external_hashes, - path, + + None, path, side.inode, side.position, ) diff --git a/atomic-repository/src/repository/status.rs b/atomic-repository/src/repository/status.rs index e33514d3..8e3e49c3 100644 --- a/atomic-repository/src/repository/status.rs +++ b/atomic-repository/src/repository/status.rs @@ -507,7 +507,8 @@ impl Repository { &inode_graph_table, &visibility, &external_hashes, - &item.path, + + None, &item.path, item.inode, item.position, ) diff --git a/libatomic/examples/agent_sandbox_bench.rs b/libatomic/examples/agent_sandbox_bench.rs index f74142c5..3273e8dc 100644 --- a/libatomic/examples/agent_sandbox_bench.rs +++ b/libatomic/examples/agent_sandbox_bench.rs @@ -709,6 +709,7 @@ mod tests { warmups: 0, allow_debug: true, keep_fixtures: false, + span: true, }) .unwrap(); let report: serde_json::Value = serde_json::from_slice(&fs::read(output).unwrap()).unwrap(); From 0860a5acb076a76d9c566cddb059d7264bc38419 Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Tue, 10 Nov 2026 00:16:58 -0600 Subject: [PATCH 8/9] chore: drop the local git working-copy baseline harness The materialize_bench comparison set is atomic-internal: paired graph/span cases and partial-vs-full ratios. Wire-protocol Git comparison stays in the agent-sandbox benchmark, where both sides start from nothing over their real transports. --- tools/materialize-bench/README.md | 30 --- tools/materialize-bench/git_baseline.py | 184 ------------------- tools/materialize-bench/test_git_baseline.py | 61 ------ 3 files changed, 275 deletions(-) delete mode 100644 tools/materialize-bench/git_baseline.py delete mode 100644 tools/materialize-bench/test_git_baseline.py diff --git a/tools/materialize-bench/README.md b/tools/materialize-bench/README.md index 14b0da8a..f4445c8d 100644 --- a/tools/materialize-bench/README.md +++ b/tools/materialize-bench/README.md @@ -75,36 +75,6 @@ favors spans. Database sizes are post-sample, uncompacted sizes, not a storage efficiency measurement. Errors and incomplete/duplicate samples fail the run; they are not passes. Fixtures are disposable; `--keep-fixtures` retains them. -## Git working-copy baseline - -`git_baseline.py` times Git doing the closest ordinary operation to a full -materialization, on a disposable Git repository built with the **same -synthetic content and edit count** as the atomic fixture (same file names, -line values, hot-file edit sequence, and one commit per edit): - -```sh -python3 -B tools/materialize-bench/git_baseline.py --profile many-files \ - --output /approved/scratch/git-01 -``` - -Cases: `checkout_full_noop` and `checkout_full_missing` (remove every -generated file, then `git checkout -- .`). Bytes, modes and a clean -`git status --porcelain` are verified after every sample outside timing. - -Boundary honesty for comparisons against a `materialize_bench` report from the -same machine and profile: - -- Git checkout restores from the **index**; it never touches pack files or - history. The Git analog of reading history is clone/fetch, which this - harness deliberately does not time. -- `total_ns` includes subprocess spawn overhead; atomic samples measure an - in-process API. Treat a few milliseconds as measurement overhead. -- Files Git considers unchanged are skipped by stat, so noop cases measure - Git's fast path, just as atomic's lease observation is its fast path. -- Reports are only comparable pairwise, same machine, same profile, close in - time; OS caches and system load remain uncontrolled. Ratios within one - report (partial against full) are more trustworthy than cross-tool ratios. - ## Agent-sandbox sync benchmark `run_agent_sandbox.py` builds and drives diff --git a/tools/materialize-bench/git_baseline.py b/tools/materialize-bench/git_baseline.py deleted file mode 100644 index 7334a262..00000000 --- a/tools/materialize-bench/git_baseline.py +++ /dev/null @@ -1,184 +0,0 @@ -#!/usr/bin/env python3 -"""Timed Git working-copy baseline for materialization comparisons. - -Builds a disposable Git repository with the same synthetic history shapes as -atomic-repository/examples/materialize_bench.rs (same files, lines, edits and -commit count), then times `git checkout --` full working-tree restores after -removing files. Bytes and modes of every file, plus a clean `git status`, are -verified after every sample outside timing. - -This measures Git's worktree-write floor: checkout restores blob content from -the index. Git history reads (clone/fetch/pack access) are a different -operation and are not measured here. subprocess spawn overhead is inside -total_ns. OS caches are uncontrolled. Compare its report only against an -atomic materialize_bench run with the same profile on the same machine. -""" -import argparse -import json -import os -from pathlib import Path -import platform -import shutil -import subprocess -import sys -import tempfile -import time - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from run import PROFILES, distribution # noqa: E402 - -CASES = ('checkout_full_noop', 'checkout_full_missing') - - -def generated_line(file_index, line_index): - return f'file_{file_index:04} line_{line_index:06} value_00000000\n' - - -def edited_line(line_index, edit): - return f'file_0000 line_{line_index:06} value_{edit + 1:08}\n' - - -def build_golden(files, lines): - golden = {} - for index in range(files): - golden[f'files/file_{index:04}.txt'] = ''.join( - generated_line(index, line) for line in range(lines)).encode() - golden['empty.txt'] = b'' - return golden - - -def git(root, *args, dates=None): - env = dict(os.environ) - env.update( - GIT_AUTHOR_NAME='bench', GIT_AUTHOR_EMAIL='bench@example.invalid', - GIT_COMMITTER_NAME='bench', GIT_COMMITTER_EMAIL='bench@example.invalid') - if dates is not None: - stamp = f'{1_600_000_000 + dates} +0000' - env.update(GIT_AUTHOR_DATE=stamp, GIT_COMMITTER_DATE=stamp) - result = subprocess.run(['git', *args], cwd=root, env=env, capture_output=True) - if result.returncode != 0: - raise RuntimeError(f'git {args[0]} failed: {result.stderr.decode(errors="replace")}') - return result.stdout - - -def build_fixture(root, files, lines, edits): - (root / 'files').mkdir(parents=True) - contents = {f'files/file_{index:04}.txt': - ''.join(generated_line(index, line) for line in range(lines)) - for index in range(files)} - for path, content in contents.items(): - (root / path).write_text(content) - (root / 'empty.txt').write_bytes(b'') - git(root, '-c', 'init.defaultBranch=dev', 'init', '-q') - git(root, 'add', '-A') - git(root, 'commit', '-q', '-m', 'synthetic baseline', dates=0) - hot = contents[f'files/file_0000.txt'].splitlines(keepends=True) - for edit in range(edits): - line = (edit * 17) % lines - hot[line] = edited_line(line, edit) - (root / 'files/file_0000.txt').write_text(''.join(hot)) - git(root, 'add', '-A') - git(root, 'commit', '-q', '-m', 'synthetic hot-file edit', dates=edit + 1) - # The golden state is the final dev tip, after every edit — the same - # capture point as the atomic fixture's "dev" golden. - golden = build_golden(files, lines) - golden['files/file_0000.txt'] = ''.join(hot).encode() - return golden - - -def verify(root, golden): - for path, content in golden.items(): - target = root / path - if not target.is_file(): - raise RuntimeError(f'missing after checkout: {path}') - if target.read_bytes() != content: - raise RuntimeError(f'bytes differ: {path}') - status = git(root, 'status', '--porcelain') - if status: - raise RuntimeError(f'working tree not clean: {status.decode(errors="replace")!r}') - - -def sample(root, golden, case): - if case == 'checkout_full_missing': - for path in sorted(golden): - (root / path).unlink() - started = time.monotonic_ns() - git(root, 'checkout', '--', '.') - total_ns = time.monotonic_ns() - started - # Filesystem equality and a clean index/status check are never timed. - verify(root, golden) - return {'case': case, 'total_ns': total_ns, 'phases_ns': {}, - 'verified_files': len(golden), - 'output_bytes': sum(len(content) for content in golden.values())} - - -def run(output, files, lines, edits, iterations, warmups, keep_fixtures): - if files <= 0 or lines < 2 or iterations < 1 or warmups < 0: - raise ValueError('invalid fixture or iteration configuration') - git_version = subprocess.run(['git', '--version'], capture_output=True, - check=True).stdout.decode().strip() - output.mkdir(parents=True, exist_ok=False) - output.chmod(0o700) - fixture = Path(tempfile.mkdtemp(dir=output, prefix='fixture-')) - metadata = {'outcome': 'incomplete', 'tool': 'git_baseline', 'schema': 1, - 'git_version': git_version, 'platform': platform.platform(), - 'logical_cpus': os.cpu_count(), - 'cache_policy': 'OS caches uncontrolled; not cold-cache claims', - 'boundary': 'git checkout restores from the index; pack/clone ' - 'history reads and subprocess spawn overhead are ' - 'inside total_ns'} - try: - golden = build_fixture(fixture, files, lines, edits) - samples = [] - for iteration in range(warmups + iterations): - for case in CASES: - measured = sample(fixture, golden, case) - if iteration >= warmups: - measured['iteration'] = iteration - warmups - samples.append(measured) - metadata.update(config=dict(files=files, lines=lines, edits=edits, - iterations=iterations, warmups=warmups), - fixture=dict(files=len(golden))) - metadata['samples'] = samples - (output / 'raw.json').write_text(json.dumps({'schema': 1, 'samples': samples}, indent=2) + '\n') - grouped = {} - for entry in samples: - grouped.setdefault(entry['case'], []).append(entry['total_ns'] / 1_000_000) - summary = {case: distribution(values) for case, values in grouped.items()} - metadata['outcome'] = 'measured' - metadata['summary_ms'] = summary - print('| Case | Median ms | Mean ms | p95 ms |') - print('|---|---:|---:|---:|') - for case in CASES: - row = summary[case] - print(f"| {case} | {row['median']:.3f} | {row['mean']:.3f} | {row['p95']:.3f} |") - print('Artifacts:', output) - except Exception as error: - metadata.update(outcome='failed-or-invalid', error=str(error)) - raise - finally: - (output / 'run.json').write_text(json.dumps(metadata, indent=2, sort_keys=True) + '\n') - if not keep_fixtures: - shutil.rmtree(fixture, ignore_errors=True) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--output', required=True, type=Path, - help='new artifact directory; must not exist') - parser.add_argument('--profile', choices=PROFILES, default='many-files') - for name in ('files', 'lines', 'edits'): - parser.add_argument('--' + name, type=int) - parser.add_argument('--iterations', type=int, default=8) - parser.add_argument('--warmups', type=int, default=1) - parser.add_argument('--keep-fixtures', action='store_true') - args = parser.parse_args() - config = dict(PROFILES[args.profile]) - config.update({name: getattr(args, name) for name in ('files', 'lines', 'edits') - if getattr(args, name) is not None}) - run(args.output, iterations=args.iterations, warmups=args.warmups, - keep_fixtures=args.keep_fixtures, **config) - - -if __name__ == '__main__': - main() diff --git a/tools/materialize-bench/test_git_baseline.py b/tools/materialize-bench/test_git_baseline.py deleted file mode 100644 index f6885e6c..00000000 --- a/tools/materialize-bench/test_git_baseline.py +++ /dev/null @@ -1,61 +0,0 @@ -import importlib.util -import json -from pathlib import Path -import shutil -import tempfile -import unittest - -spec = importlib.util.spec_from_file_location('git_baseline', Path(__file__).with_name('git_baseline.py')) -git_baseline = importlib.util.module_from_spec(spec) -spec.loader.exec_module(git_baseline) - -GIT = shutil.which('git') is not None - - -@unittest.skipUnless(GIT, 'git executable required') -class GitBaselineTests(unittest.TestCase): - def test_generated_content_matches_atomic_fixture_format(self): - golden = git_baseline.build_golden(3, 8) - self.assertEqual( - golden['files/file_0002.txt'], - ''.join(git_baseline.generated_line(2, line) for line in range(8)).encode()) - self.assertEqual(git_baseline.edited_line(3, 1), - 'file_0000 line_000003 value_00000002\n') - - def test_fixture_history_and_clean_status(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / 'repo' - root.mkdir() - golden = git_baseline.build_fixture(root, 3, 8, 2) - self.assertEqual(golden['files/file_0000.txt'].count(b'value_00000001'), 1) - git_baseline.verify(root, golden) - - def test_smoke_run_measures_every_case_and_verifies(self): - with tempfile.TemporaryDirectory() as tmp: - output = Path(tmp) / 'out' - git_baseline.run(output, files=3, lines=8, edits=2, - iterations=2, warmups=1, keep_fixtures=False) - raw = json.loads((output / 'raw.json').read_text()) - cases = {(s['case'], s['iteration']) for s in raw['samples']} - self.assertEqual(cases, {(case, i) for case in git_baseline.CASES - for i in range(2)}) - for entry in raw['samples']: - self.assertGreater(entry['total_ns'], 0) - run_report = json.loads((output / 'run.json').read_text()) - self.assertEqual(run_report['outcome'], 'measured') - self.assertEqual(len(run_report['summary_ms']), len(git_baseline.CASES)) - - def test_corrupted_fixture_fails_verification(self): - if not GIT: - self.skipTest('git executable required') - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / 'repo' - root.mkdir() - golden = git_baseline.build_fixture(root, 2, 4, 1) - (root / 'files/file_0000.txt').write_text('corrupted\n') - with self.assertRaises(RuntimeError): - git_baseline.verify(root, golden) - - -if __name__ == '__main__': - unittest.main() From 9ce945ca758d58b0060e2109a09f65e3b96359ac Mon Sep 17 00:00:00 2001 From: Aaron Ogle Date: Tue, 10 Nov 2026 00:28:21 -0600 Subject: [PATCH 9/9] feat: COW sandbox provision case for the agent-sandbox benchmark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit atomic_sandbox_cow times Repository::provision_sandbox on the same synthetic fixture: the local copy-on-write clone an agent gets on a machine that already holds the repository. Measured (2,048 files, 11 MB, reflink on APFS): 276 ms median — per-file reflink syscall overhead dominates at ~0.13 ms/file, so provision time scales with file count, not bytes. Beats the RPC partial (336 ms) and git clone (779 ms); the whole-tree clonefile(2) fast path is the follow-up that would make it near-instant. --- libatomic/examples/agent_sandbox_bench.rs | 66 +++++++++++++------- tools/materialize-bench/run_agent_sandbox.py | 2 +- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/libatomic/examples/agent_sandbox_bench.rs b/libatomic/examples/agent_sandbox_bench.rs index 3273e8dc..9aeb3771 100644 --- a/libatomic/examples/agent_sandbox_bench.rs +++ b/libatomic/examples/agent_sandbox_bench.rs @@ -13,6 +13,10 @@ //! directories may be streamed. //! - `git_clone_full` — `git clone` of a bare repository over the `file://` //! wire transport. +//! - `atomic_sandbox_cow` — `Repository::provision_sandbox`, the local +//! copy-on-write clone an agent gets on a machine that already holds the +//! repository (reflink where the filesystem supports it). No wire, no +//! RPC: the instant-workspace path. //! //! The atomic side exercises the real transport end to end: a fresh channel //! per sample (a sandbox never reuses a connection), the generated client, @@ -50,6 +54,7 @@ use hyper_util::rt::TokioIo; const CASES: &[&str] = &[ "atomic_rpc_full", "atomic_rpc_partial", + "atomic_sandbox_cow", "git_clone_full", ]; @@ -590,7 +595,7 @@ fn run(config: Config) -> Result<()> { let handle = state.register(host_root.clone()); let repository = handle.repository_ref().repository_id; handle.repository()?; // fail fast; the server would serve errors otherwise - let server = serve(state, host_root, socket.clone())?; + let server = serve(state, host_root.clone(), socket.clone())?; let east = east_paths(&config); let east_bytes: usize = golden .iter() @@ -604,29 +609,44 @@ fn run(config: Config) -> Result<()> { for case in CASES { let partial = case.contains("partial"); let sandbox = fixtures.path().join(format!("sandbox-{case}-{iteration}")); + // Store open is setup, not measured: the COW clone is the + // operation under test. + let cow_repo = (*case == "atomic_sandbox_cow").then(|| { + atomic_repository::Repository::open_existing_wait( + &host_root, + std::time::Duration::from_secs(30), + ) + .expect("open host repository") + }); let started = Instant::now(); - let (phases, entries) = if case.starts_with("atomic") { - let target = libatomic::atomic::ViewRef { - view_id: Vec::new(), - name: Some("dev".into()), - }; - let reference = libatomic::atomic::RepositoryRef { - authority: "local".into(), - repository_id: repository.clone(), - workspace_id: None, - }; - let (phases, entries) = atomic_sandbox( - &socket, - reference, - target, - partial.then(|| PARTIAL_PREFIX.to_string()), - &sandbox, - )?; - (phases, Some(entries)) - } else if *case == "git_clone_full" { - (git_clone_full(&bare, &sandbox)?, None) - } else { - bail!("unknown case: {case}") + let (phases, entries) = match *case { + "atomic_rpc_full" | "atomic_rpc_partial" => { + let target = libatomic::atomic::ViewRef { + view_id: Vec::new(), + name: Some("dev".into()), + }; + let reference = libatomic::atomic::RepositoryRef { + authority: "local".into(), + repository_id: repository.clone(), + workspace_id: None, + }; + let (phases, entries) = atomic_sandbox( + &socket, + reference, + target, + partial.then(|| PARTIAL_PREFIX.to_string()), + &sandbox, + )?; + (phases, Some(entries)) + } + "atomic_sandbox_cow" => { + let repo = cow_repo.as_ref().unwrap(); + let wc = repo.require_working_copy_id()?; + let written = repo.provision_sandbox(wc, &sandbox, "dev")?; + (BTreeMap::new(), Some(written)) + } + "git_clone_full" => (git_clone_full(&bare, &sandbox)?, None), + _ => bail!("unknown case: {case}"), }; let total_ns = started.elapsed().as_nanos(); // Byte verification is never timed. diff --git a/tools/materialize-bench/run_agent_sandbox.py b/tools/materialize-bench/run_agent_sandbox.py index 29e9491f..0449c61a 100644 --- a/tools/materialize-bench/run_agent_sandbox.py +++ b/tools/materialize-bench/run_agent_sandbox.py @@ -29,7 +29,7 @@ 'many-files': dict(files_east=8, files_west=16, lines=128, edits=24), 'hot-file': dict(files_east=1, files_west=16, lines=4096, edits=48), } -CASES = ('atomic_rpc_full', 'atomic_rpc_partial', 'git_clone_full') +CASES = ('atomic_rpc_full', 'atomic_rpc_partial', 'atomic_sandbox_cow', 'git_clone_full') def validate_report(report, iterations):