diff --git a/atomic-cli/src/commands/git/checkpoint.rs b/atomic-cli/src/commands/git/checkpoint.rs index 26b675e4..02d98c25 100644 --- a/atomic-cli/src/commands/git/checkpoint.rs +++ b/atomic-cli/src/commands/git/checkpoint.rs @@ -344,26 +344,71 @@ fn observe_atomic_on_handle( .map_err(|error| CheckpointError::Atomic(error.to_string()))? .map(|(object, markers)| (object.hash().ok(), markers)); + // Paths with marker bytes never consult the graph. Every other path is + // streamed from one read (one projection, one span context) in the same + // sorted order, and hashed as it arrives so only one file is held at a + // time; marker paths are hashed from their marker bytes in between. + let marker_bytes = |path: &str| -> Option<&[u8]> { + conflict_markers + .as_ref() + .and_then(|(_, markers)| markers.get(path)) + .map(Vec::as_slice) + }; + let graph_paths: Vec = paths + .iter() + .filter(|path| marker_bytes(path).is_none()) + .cloned() + .collect(); let mut hasher = blake3::Hasher::new(); hasher.update(ATOMIC_MANIFEST_DOMAIN); - for path in paths { - let content = match &conflict_markers { - Some((_, markers)) if markers.contains_key(&path) => { - markers.get(&path).cloned().expect("marker bytes") - } - _ => repo - .get_file_content_on_view(&path, view) - .map_err(|error| CheckpointError::Atomic(error.to_string()))? - .ok_or_else(|| { - CheckpointError::Atomic(format!( - "visible path '{path}' has no graph-derived content on view '{view}'" - )) - })?, - }; + let hash_entry = |hasher: &mut blake3::Hasher, path: &str, content: &[u8]| { hasher.update(&(path.len() as u64).to_le_bytes()); hasher.update(path.as_bytes()); hasher.update(&(content.len() as u64).to_le_bytes()); - hasher.update(&content); + hasher.update(content); + }; + let mut next = 0usize; + let mut missing: Option = None; + let streamed = repo.for_each_file_content_on_view(&graph_paths, view, |path, bytes| { + while paths.get(next).is_some_and(|pending| pending != path) { + let pending = &paths[next]; + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + missing = Some(pending.clone()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{pending}' has no content"), + }); + } + } + next += 1; + } + let Some(content) = bytes else { + missing = Some(path.to_string()); + return Err(atomic_repository::RepositoryError::InvalidOperation { + message: format!("visible path '{path}' has no content"), + }); + }; + hash_entry(&mut hasher, path, &content); + next += 1; + Ok(()) + }); + if let Some(path) = missing { + return Err(CheckpointError::Atomic(format!( + "visible path '{path}' has no graph-derived content on view '{view}'" + ))); + } + streamed.map_err(|error| CheckpointError::Atomic(error.to_string()))?; + while let Some(pending) = paths.get(next) { + match marker_bytes(pending) { + Some(marker) => hash_entry(&mut hasher, pending, marker), + None => { + return Err(CheckpointError::Atomic(format!( + "visible path '{pending}' has no graph-derived content on view '{view}'" + ))) + } + } + next += 1; } Ok(CheckpointAtomicState { diff --git a/atomic-cli/src/commands/restore.rs b/atomic-cli/src/commands/restore.rs index 544db13e..97fb0dad 100644 --- a/atomic-cli/src/commands/restore.rs +++ b/atomic-cli/src/commands/restore.rs @@ -55,11 +55,12 @@ //! Would restore: src/lib.rs //! ``` +use std::collections::HashMap; use std::path::{Path, PathBuf}; use atomic_repository::tracking::TrackingOptions; use atomic_repository::{ - FileStatus, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, + FileStatus, RecordedPresence, Repository, RepositoryStatus, StatusOptions, WorkspaceTxnMode, }; use clap::Parser; @@ -247,6 +248,117 @@ impl Restore { } } + /// Classify and restore every Modified/Deleted path in one pass, returning + /// the outcome for each path so the caller can report them in status order. + /// + /// One projection decides which paths the view still records, and one + /// materialization writes all of them, instead of rendering and + /// materializing file by file. Outcomes match the single-file path: a path + /// the view does not project (or projects as a directory) is `Skipped`, + /// and a path with an unresolved name conflict fails with the same error. + /// A projection failure is reported for every path as the single-file read + /// would have reported it; if the batched materialization fails, every + /// path falls back to the single-file restore so outcomes stay per file. + fn restore_contents( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> HashMap> { + let mut outcomes = HashMap::new(); + if paths.is_empty() { + return outcomes; + } + let names: Vec = paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(); + let presence = match repo.recorded_presence_on_view(&names, view) { + Ok(presence) => presence, + Err(error) => { + // A view or projection error would fail every single-path read + // the same way; report it per file without re-projecting. + for path in paths { + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); + } + return outcomes; + } + }; + let mut selected: Vec<(PathBuf, String)> = Vec::new(); + for ((path, name), presence) in paths.iter().zip(&names).zip(presence) { + match presence { + RecordedPresence::Present => selected.push((path.clone(), name.clone())), + RecordedPresence::Absent => { + outcomes.insert(path.clone(), Ok(RestoreOutcome::Skipped)); + } + RecordedPresence::NameConflict { paths, claimants } => { + let error = atomic_repository::name_conflict_error( + &atomic_repository::normalize_path(path), + paths, + claimants, + ); + outcomes.insert( + path.clone(), + Err(CliError::Internal(anyhow::anyhow!( + "Failed to get file content: {}", + error + ))), + ); + } + } + } + if selected.is_empty() { + return outcomes; + } + let batch = selected.iter().map(|(_, name)| name.clone()).collect(); + match repo.materialize_paths(working_copy, batch) { + Ok(_) => { + for (path, _) in selected { + outcomes.insert(path, Ok(RestoreOutcome::Restored)); + } + } + Err(error) => { + print_warning(&format!( + "Batched restore failed ({}); restoring file by file", + error + )); + let retry: Vec = selected.into_iter().map(|(path, _)| path).collect(); + outcomes.extend(self.restore_contents_one_by_one(repo, working_copy, view, &retry)); + } + } + outcomes + } + + fn restore_contents_one_by_one( + &self, + repo: &Repository, + working_copy: atomic_core::WorkingCopyId, + view: &str, + paths: &[PathBuf], + ) -> HashMap> { + paths + .iter() + .map(|path| { + let outcome = self.restore_file( + repo, + working_copy, + view, + Path::new(""), + path, + FileStatus::Modified, + ); + (path.clone(), outcome) + }) + .collect() + } + /// Restore a single file according to its status. /// /// - `Added` (tracked but not recorded): undo tracking. The file stays @@ -455,17 +567,39 @@ impl Command for Restore { let mut restored_count = 0; let mut error_count = 0; + // Modified/Deleted files are classified and materialized together so + // the view is projected and its span history decoded once, not once + // per file. Added files are untracked one by one. Outcomes are + // reported in status order, as before. + let content_paths: Vec = files_to_restore + .iter() + .filter(|(_, file_status)| *file_status != FileStatus::Added) + .map(|(path, _)| path.clone()) + .collect(); + let mut content_outcomes = + self.restore_contents(&repo, working_copy, &workspace_view, &content_paths); + for (path, file_status) in &files_to_restore { let path_display = path.display(); - match self.restore_file( - &repo, - working_copy, - &workspace_view, - &repo_root, - path, - *file_status, - ) { + let outcome = if *file_status == FileStatus::Added { + self.restore_file( + &repo, + working_copy, + &workspace_view, + &repo_root, + path, + *file_status, + ) + } else { + content_outcomes.remove(path).unwrap_or_else(|| { + Err(CliError::Internal(anyhow::anyhow!( + "no restore outcome recorded for '{}'", + path_display + ))) + }) + }; + match outcome { Ok(RestoreOutcome::Restored) => { println!(" Restored: {}", path_display); restored_count += 1; diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index fa03ff60..34bee428 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -23,6 +23,28 @@ fn graph_visibility_at_change( graph_visibility_from_membership(txn, &membership).map(Some) } +/// Whether a view projects a file at a path, without rendering its content. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecordedPresence { + /// The view projects a non-directory entry at this path. + Present, + /// The path is absent on the view, or projects a directory. + Absent, + /// The path has an unresolved name conflict; no single content applies. + NameConflict { paths: usize, claimants: usize }, +} + +/// The error every content read returns for a path with an unresolved name +/// conflict. Callers that classify paths first must report the same error. +pub fn name_conflict_error(path: &str, paths: usize, claimants: usize) -> RepositoryError { + RepositoryError::InvalidOperation { + message: format!( + "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", + path, paths, claimants + ), + } +} + impl Repository { /// Get the recorded content for a tracked file. /// @@ -289,6 +311,52 @@ impl Repository { } } + /// Decide, for many paths at once, whether a view projects a file there. + /// + /// One transaction and one tree projection answer every path; nothing is + /// rendered. The rules match [`Self::get_materialized_entry_on_view`]: + /// a visible name conflict is reported instead of choosing content, and a + /// directory or an absent path is `Absent`. + pub fn recorded_presence_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result, RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + Ok(paths + .iter() + .map(|path| { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return RecordedPresence::NameConflict { + paths: conflict.paths.len(), + claimants: conflict.sides.len(), + }; + } + match projection.present.get(&normalized) { + Some(item) if !item.is_directory => RecordedPresence::Present, + _ => RecordedPresence::Absent, + } + }) + .collect()) + } + /// Resolve a file as an explicit present/absent materialization entry on a view. /// /// Presence comes from the operation-aware path lifecycle projection. Empty @@ -315,14 +383,11 @@ impl Repository { let projection = self.project_tree_for_visibility_scoped(&txn, &visibility, Some(normalized.as_str()))?; if let Some(conflict) = projection.name_conflicts.get(&normalized) { - return Err(RepositoryError::InvalidOperation { - message: format!( - "path '{}' has an unresolved name conflict ({} path(s), {} claimant(s)); refusing to choose content", - normalized, - conflict.paths.len(), - conflict.sides.len() - ), - }); + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); } let Some(item) = projection.present.get(&normalized) else { let inode = projection @@ -335,20 +400,84 @@ impl Repository { if item.is_directory { return Ok(MaterializedEntry::absent(normalized, Some(item.inode))); } - let bytes = - match retrieve_span_if_present(&txn, &self.change_store, item.inode, &visibility)? { - Some(bytes) => bytes, - None => retrieve_content_with_filter_fast( + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + let bytes = render_projected_file(&txn, &self.change_store, &spans, item, &visibility)?; + Ok(MaterializedEntry::present(normalized, item.inode, bytes)) + } + + /// Stream the recorded bytes of many files on a view through one + /// transaction, one tree projection and one shared span context. + /// + /// `visit` receives each path in the order given, with exactly what + /// [`Self::get_file_content_on_view`] returns for it: `None` for an absent + /// path or a directory. Bytes are handed over one file at a time and + /// dropped by the visitor, so peak memory stays at one file, not the whole + /// set. A requested path with an unresolved name conflict fails the call, + /// as the single-path read refuses to choose content for it; a visitor + /// error stops the walk and is returned unchanged. Nothing survives this + /// call's transaction; the visitor must not mutate repository state. + pub fn for_each_file_content_on_view( + &self, + paths: &[String], + view_name: &str, + mut visit: impl FnMut(&str, Option>) -> Result<(), RepositoryError>, + ) -> Result<(), RepositoryError> { + let txn = self + .pristine + .read_txn() + .map_err(|e| RepositoryError::Database(e.to_string()))?; + let view = txn + .get_view(view_name) + .map_err(|e| RepositoryError::Database(e.to_string()))? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let wanted: std::collections::HashSet = paths + .iter() + .map(|path| normalize_path(Path::new(path))) + .collect(); + let projection = + self.project_tree_for_visibility_wanted(&txn, &visibility, Some(&wanted))?; + let spans = SharedSpanRender::new(&txn, &self.change_store, &visibility); + for path in paths { + let normalized = normalize_path(Path::new(path)); + if let Some(conflict) = projection.name_conflicts.get(&normalized) { + return Err(name_conflict_error( + &normalized, + conflict.paths.len(), + conflict.sides.len(), + )); + } + let bytes = match projection.present.get(&normalized) { + Some(item) if !item.is_directory => Some(render_projected_file( &txn, &self.change_store, - item.inode, - item.position, - atomic_core::output::alive::RetrieveOptions::new() - .with_graph_visibility(visibility), - ) - .map_err(|error| RepositoryError::Database(error.to_string()))?, + &spans, + item, + &visibility, + )?), + _ => None, }; - Ok(MaterializedEntry::present(normalized, item.inode, bytes)) + visit(path, bytes)?; + } + Ok(()) + } + + /// Collected form of [`Self::for_each_file_content_on_view`]: one entry per + /// requested path, in order. This holds every rendered file at once, so + /// callers that only need to consume each file should stream instead. + pub fn get_file_contents_on_view( + &self, + paths: &[String], + view_name: &str, + ) -> Result>>, RepositoryError> { + let mut contents = Vec::with_capacity(paths.len()); + self.for_each_file_content_on_view(paths, view_name, |_, bytes| { + contents.push(bytes); + Ok(()) + })?; + Ok(contents) } /// Read a path through an exact assembly visibility: `view`'s validated @@ -907,6 +1036,37 @@ impl Repository { } } +/// Recorded bytes of one projected file: its span rendering when the file is +/// span-rooted on this view, otherwise the filtered graph rendering. +fn render_projected_file( + txn: &T, + store: &C, + spans: &SharedSpanRender<'_, T, C>, + item: &atomic_core::output::repo::OutputItem, + visibility: &GraphVisibilityClosure, +) -> Result, RepositoryError> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::GraphTxnT + + atomic_core::pristine::InodeGraphOps + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + if let Some(bytes) = spans.render_if_present(item.inode)? { + return Ok(bytes); + } + retrieve_content_with_filter_fast( + txn, + store, + item.inode, + item.position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(visibility.clone()), + ) + .map_err(|error| RepositoryError::Database(error.to_string())) +} + pub(super) fn retrieve_span_if_present( txn: &T, store: &C, @@ -928,6 +1088,72 @@ where .map_err(|error| RepositoryError::Database(error.to_string())) } +/// A span renderer shared by every file read through one transaction and +/// visibility closure. +/// +/// Building a [`SpanRenderContext`] verifies and decodes the whole visible +/// closure, so callers that render many files in one transaction must not +/// rebuild it per file. The context is built lazily on the first span-rooted +/// file and reused for the rest; graph-origin files never trigger a build. +/// The per-file root, capability and visibility checks are unchanged. +pub(super) struct SharedSpanRender< + 'a, + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn, + C, +> { + txn: &'a T, + store: &'a C, + visibility: &'a GraphVisibilityClosure, + context: + std::cell::OnceCell>, +} + +impl<'a, T, C> SharedSpanRender<'a, T, C> +where + T: atomic_core::pristine::span_patch_storage::SpanPatchReadTxn + + atomic_core::pristine::TreeTxnT + + atomic_core::pristine::CapabilityTxnT, + C: atomic_core::change::ChangeStore, +{ + pub(super) fn new(txn: &'a T, store: &'a C, visibility: &'a GraphVisibilityClosure) -> Self { + Self { + txn, + store, + visibility, + context: std::cell::OnceCell::new(), + } + } + + /// Render `inode` through the shared context when its span root is visible; + /// `None` means the file is graph-origin and must be read from the graph. + pub(super) fn render_if_present( + &self, + inode: Inode, + ) -> Result>, RepositoryError> { + if !has_visible_span_root(self.txn, self.store, inode, self.visibility)? { + return Ok(None); + } + let context = match self.context.get() { + Some(context) => context, + None => { + let built = atomic_core::pristine::span_patch_storage::SpanRenderContext::new( + self.txn, + self.store, + self.visibility, + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + // A concurrent `set` is impossible: `OnceCell` is single-threaded. + let _ = self.context.set(built); + self.context.get().expect("span context was just set") + } + }; + context + .render(inode) + .map(Some) + .map_err(|error| RepositoryError::Database(error.to_string())) + } +} + pub(super) fn retrieve_span_bounded_if_present( txn: &T, store: &C, diff --git a/atomic-repository/src/repository/deferred_tree.rs b/atomic-repository/src/repository/deferred_tree.rs index cc2482ae..3b7725e6 100644 --- a/atomic-repository/src/repository/deferred_tree.rs +++ b/atomic-repository/src/repository/deferred_tree.rs @@ -1051,6 +1051,31 @@ impl Repository { visibility: &GraphVisibilityClosure, wanted: Option<&str>, ) -> Result + where + T: GraphTxnT + + TreeTxnT + + PathClaimTxnT + + atomic_core::pristine::InodeGraphOps, + { + match wanted { + Some(path) => self.project_tree_for_visibility_wanted( + txn, + visibility, + Some(&HashSet::from([path.to_string()])), + ), + None => self.project_tree_for_visibility_wanted(txn, visibility, None), + } + } + + /// Set-scoped variant of [`Self::project_tree_for_visibility_scoped`]: + /// the absent-liveness check runs only for the `wanted` paths, so a + /// batch of path reads costs what the same reads cost one by one. + pub(super) fn project_tree_for_visibility_wanted( + &self, + txn: &T, + visibility: &GraphVisibilityClosure, + wanted: Option<&HashSet>, + ) -> Result where T: GraphTxnT + TreeTxnT @@ -1105,7 +1130,7 @@ impl Repository { let mut absent_check_ms = 0u128; let visible_changes = visibility.iter_dependency_first().copied().collect(); for absent in reduced.absent { - let wanted_here = wanted.is_none_or(|path| path == absent.path.as_str()); + let wanted_here = wanted.is_none_or(|paths| paths.contains(absent.path.as_str())); if wanted_here && !absent.directory && !alive_inodes.contains(&absent.inode) diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 51f727eb..f5cb2fef 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -44,6 +44,7 @@ use std::sync::Arc; use atomic_core::change::{Change, ChangeHeader, GraphOp}; pub use atomic_core::output::repo::MaterializedEntry; use atomic_core::output::repo::{MaterializeResult, OutputItem}; +pub use content::{name_conflict_error, RecordedPresence}; use atomic_core::pristine::{ GraphTxnT, GraphVisibilityClosure, MutTxnT, Pristine, TreeTxnT, ViewMembershipSet, ViewScope, diff --git a/atomic-repository/src/repository/project_tree.rs b/atomic-repository/src/repository/project_tree.rs index 1a945254..62a54fd6 100644 --- a/atomic-repository/src/repository/project_tree.rs +++ b/atomic-repository/src/repository/project_tree.rs @@ -1229,6 +1229,7 @@ impl Repository { ))); } let mut entries = Vec::new(); + let spans = super::content::SharedSpanRender::new(&txn, &self.change_store, &closure); for item in projection .present .values() @@ -1261,13 +1262,9 @@ impl Repository { ))); } let materialization = attrs.materialization; - let bytes = match super::content::retrieve_span_if_present( - &txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + let bytes = match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( @@ -1445,6 +1442,7 @@ impl Repository { } let mut entries = Vec::new(); let mut projected_paths = std::collections::BTreeSet::new(); + let spans = super::content::SharedSpanRender::new(txn, &self.change_store, &closure); for item in projection .present .values() @@ -1468,13 +1466,9 @@ impl Repository { let materialization = attrs.materialization; let bytes = match marker_bytes.get(item.path.as_str()) { Some(override_bytes) => override_bytes.clone(), - None => match super::content::retrieve_span_if_present( - txn, - &self.change_store, - item.inode, - &closure, - ) - .map_err(ProjectTreeError::from)? + None => match spans + .render_if_present(item.inode) + .map_err(ProjectTreeError::from)? { Some(content) => content, None => super::content::retrieve_content_with_filter_fast( diff --git a/atomic-repository/src/repository/record.rs b/atomic-repository/src/repository/record.rs index c4e13d0b..f968b06c 100644 --- a/atomic-repository/src/repository/record.rs +++ b/atomic-repository/src/repository/record.rs @@ -4694,11 +4694,13 @@ fn plan_file_moves( basis: MoveBasis, } + // One span context serves every deleted candidate in this planning pass. + let spans = super::content::SharedSpanRender::new(cached, store, visibility); let load_old_content = |inode: Inode, position: Position| -> Result, RecordError> { - if let Some(bytes) = - super::content::retrieve_span_if_present(cached, store, inode, visibility) - .map_err(RecordError::Repository)? + if let Some(bytes) = spans + .render_if_present(inode) + .map_err(RecordError::Repository)? { return Ok(bytes); } diff --git a/atomic-repository/src/repository/tests/batch_content_tests.rs b/atomic-repository/src/repository/tests/batch_content_tests.rs new file mode 100644 index 00000000..18903e9a --- /dev/null +++ b/atomic-repository/src/repository/tests/batch_content_tests.rs @@ -0,0 +1,189 @@ +use super::*; +use atomic_core::pristine::{CapabilityMutTxnT, MutTxnT, SPAN_RANGE_CAPABILITY}; + +fn record_all(repo: &TestRepository, message: &str) { + repo.record( + ChangeHeader::new(message), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true) + .sync_vault(false) + .enrich_kg(false), + ) + .unwrap(); +} + +fn enable_spans(repo: &TestRepository) { + let mut txn = repo.pristine().write_txn().unwrap(); + txn.put_required_capability(SPAN_RANGE_CAPABILITY).unwrap(); + txn.commit().unwrap(); +} + +fn write_and_add(temp: &tempfile::TempDir, repo: &mut TestRepository, path: &str, bytes: &[u8]) { + let full = temp.path().join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(full, bytes).unwrap(); + repo.add(path, TrackingOptions::default()).unwrap(); +} + +/// Batched reads must be byte-identical to the single-path reads they replace, +/// for graph-origin files, span-rooted files, empty files, directories and +/// absent paths, on every view they are asked about. +#[test] +fn batch_reads_and_presence_match_single_path_reads() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "legacy.txt", b"legacy\n"); + record_all(&repo, "graph origin"); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "empty.txt", b""); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "new files"); + std::fs::write(temp.path().join("b.txt"), b"beta edited\n").unwrap(); + record_all(&repo, "edit b"); + + repo.create_view_from("feature", "dev").unwrap(); + repo.switch_view("feature").unwrap(); + std::fs::write(temp.path().join("a.txt"), b"alpha on feature\n").unwrap(); + record_all(&repo, "feature edit"); + + if spans { + use atomic_core::pristine::span_patch_storage::SpanPatchReadTxn; + use atomic_core::pristine::TreeTxnT; + let txn = repo.pristine().read_txn().unwrap(); + let inode = txn.get_inode("a.txt").unwrap().unwrap(); + assert!( + txn.span_root(inode).unwrap().is_some(), + "fixture must record a.txt as a span-rooted file" + ); + let legacy = txn.get_inode("legacy.txt").unwrap().unwrap(); + assert!( + txn.span_root(legacy).unwrap().is_none(), + "fixture must keep legacy.txt graph-origin" + ); + } + + let paths: Vec = [ + "a.txt", + "b.txt", + "empty.txt", + "legacy.txt", + "dir/c.txt", + "dir", + "missing.txt", + ] + .map(str::to_string) + .into(); + for view in ["dev", "feature"] { + let batch = repo.get_file_contents_on_view(&paths, view).unwrap(); + assert_eq!(batch.len(), paths.len()); + for (path, bytes) in paths.iter().zip(&batch) { + assert_eq!( + bytes, + &repo.get_file_content_on_view(path, view).unwrap(), + "spans={spans} view={view} path={path}" + ); + } + let expected_a: &[u8] = if view == "feature" { + b"alpha on feature\n" + } else { + b"alpha\n" + }; + assert_eq!(batch[0].as_deref(), Some(expected_a)); + assert_eq!(batch[1].as_deref(), Some(b"beta edited\n".as_slice())); + assert_eq!(batch[2].as_deref(), Some(b"".as_slice())); + assert_eq!(batch[3].as_deref(), Some(b"legacy\n".as_slice())); + assert_eq!(batch[4].as_deref(), Some(b"gamma\n".as_slice())); + assert_eq!(batch[5], None, "directories have no content"); + assert_eq!(batch[6], None, "absent paths have no content"); + + let presence = repo.recorded_presence_on_view(&paths, view).unwrap(); + let expected: Vec = batch + .iter() + .map(|bytes| { + if bytes.is_some() { + RecordedPresence::Present + } else { + RecordedPresence::Absent + } + }) + .collect(); + assert_eq!(presence, expected, "spans={spans} view={view}"); + } + } +} + +/// An unknown view fails the batch exactly like the single-path read. +#[test] +fn batch_reads_reject_unknown_views() { + let (temp, mut repo) = create_temp_repo(); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + record_all(&repo, "one file"); + let paths = vec!["a.txt".to_string()]; + assert!(matches!( + repo.get_file_contents_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert!(matches!( + repo.recorded_presence_on_view(&paths, "nope"), + Err(RepositoryError::ViewNotFound { .. }) + )); + assert_eq!(repo.get_file_contents_on_view(&[], "dev").unwrap(), vec![]); +} + +/// The streaming read visits every path in the requested order with the same +/// bytes the collected read returns, and a visitor error stops the walk and +/// comes back unchanged. +#[test] +fn streaming_reads_visit_in_order_and_propagate_visitor_errors() { + for spans in [false, true] { + let (temp, mut repo) = create_temp_repo(); + if spans { + enable_spans(&repo); + } + write_and_add(&temp, &mut repo, "b.txt", b"beta\n"); + write_and_add(&temp, &mut repo, "a.txt", b"alpha\n"); + write_and_add(&temp, &mut repo, "dir/c.txt", b"gamma\n"); + record_all(&repo, "files"); + let paths: Vec = ["dir/c.txt", "missing.txt", "a.txt", "dir", "b.txt"] + .map(str::to_string) + .into(); + let collected = repo.get_file_contents_on_view(&paths, "dev").unwrap(); + let mut visited = Vec::new(); + repo.for_each_file_content_on_view(&paths, "dev", |path, bytes| { + visited.push((path.to_string(), bytes)); + Ok(()) + }) + .unwrap(); + let visited_paths: Vec<&str> = visited.iter().map(|(path, _)| path.as_str()).collect(); + assert_eq!( + visited_paths, + paths.iter().map(String::as_str).collect::>() + ); + let visited_bytes: Vec>> = + visited.into_iter().map(|(_, bytes)| bytes).collect(); + assert_eq!(visited_bytes, collected, "spans={spans}"); + + let mut seen = Vec::new(); + let error = repo + .for_each_file_content_on_view(&paths, "dev", |path, _| { + seen.push(path.to_string()); + if path == "a.txt" { + return Err(RepositoryError::InvalidOperation { + message: "visitor refused".into(), + }); + } + Ok(()) + }) + .unwrap_err(); + assert!(error.to_string().contains("visitor refused"), "{error}"); + assert_eq!(seen, ["dir/c.txt", "missing.txt", "a.txt"], "spans={spans}"); + } +} diff --git a/atomic-repository/src/repository/tests/mod.rs b/atomic-repository/src/repository/tests/mod.rs index 80d3a470..0a505b6f 100644 --- a/atomic-repository/src/repository/tests/mod.rs +++ b/atomic-repository/src/repository/tests/mod.rs @@ -9,6 +9,7 @@ use std::path::Path; use tempfile::TempDir; mod attribute_tests; +mod batch_content_tests; mod binding_fetch_tests; mod binding_store_tests; mod binding_verify_tests; diff --git a/atomic-repository/src/repository/verify.rs b/atomic-repository/src/repository/verify.rs index b41df533..67f6c92f 100644 --- a/atomic-repository/src/repository/verify.rs +++ b/atomic-repository/src/repository/verify.rs @@ -138,6 +138,38 @@ impl Repository { // Files visible (tracked + recorded) on the current view. let visible = self.visible_file_paths(&desired_view)?; + // Clean files are compared against the graph. Stream them through one + // transaction so the view's span history is decoded once, comparing + // each file as it arrives; only the mismatching lengths are kept and + // reported below in the visible-path order. + let is_uncommitted_or_conflicted = |path: &String| { + matches!( + status_by_path.get(path), + Some(FileStatus::Modified) + | Some(FileStatus::Added) + | Some(FileStatus::Deleted) + | Some(FileStatus::Conflicted) + ) + }; + let clean_paths: Vec = visible + .iter() + .filter(|path| !is_uncommitted_or_conflicted(path)) + .cloned() + .collect(); + let mut drift: HashMap = HashMap::new(); + let mut clean_files_checked = 0usize; + self.for_each_file_content_on_view(&clean_paths, &desired_view, |path, graph| { + let disk = std::fs::read(self.root.join(path)).ok(); + if let (Some(disk), Some(graph)) = (disk, graph) { + clean_files_checked += 1; + if disk != graph { + drift.insert(path.to_string(), (disk.len(), graph.len())); + } + } + Ok(()) + })?; + report.clean_files_checked += clean_files_checked; + for path in &visible { let st = status_by_path.get(path).copied(); @@ -154,18 +186,12 @@ impl Repository { // Clean (absent from status) or other benign states: the disk // must match the graph exactly. _ => { - let abs = self.root.join(path); - let disk = std::fs::read(&abs).ok(); - let graph = self.get_file_content_on_view(path, &desired_view)?; - if let (Some(disk), Some(graph)) = (disk.as_ref(), graph.as_ref()) { - report.clean_files_checked += 1; - if disk != graph { - report.problems.push(VerifyProblem::MaterializationDrift { - path: path.clone(), - disk_len: disk.len(), - graph_len: graph.len(), - }); - } + if let Some((disk_len, graph_len)) = drift.remove(path.as_str()) { + report.problems.push(VerifyProblem::MaterializationDrift { + path: path.clone(), + disk_len, + graph_len, + }); } } } diff --git a/libatomic/src/daemon/services.rs b/libatomic/src/daemon/services.rs index e7c69040..ea937f63 100644 --- a/libatomic/src/daemon/services.rs +++ b/libatomic/src/daemon/services.rs @@ -2043,6 +2043,34 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI name paths explicitly or pass force", )); } + // One projection decides which Modified/Deleted paths the view + // still records; rendering every file here only to discard the + // bytes would decode the view's history once per file. A + // projection error therefore fails the request before any path is + // untracked, and a render error now surfaces from materialization + // (which plans before it writes). The walk below keeps status + // order, so untracking and a name-conflict failure happen at the + // same position as before. + let content_paths: Vec = dirty + .iter() + .filter(|(_, file_status)| { + *file_status != atomic_repository::status::FileStatus::Added + }) + .map(|(path, _)| path.clone()) + .collect(); + let presence: std::collections::HashMap = + if content_paths.is_empty() { + Default::default() + } else { + content_paths + .iter() + .cloned() + .zip( + repo.recorded_presence_on_view(&content_paths, repo.current_view()) + .map_err(repository_error)?, + ) + .collect() + }; let mut restored: Vec = Vec::new(); let mut materialize = std::collections::HashSet::new(); for (path, file_status) in dirty { @@ -2058,16 +2086,26 @@ impl repository_mutation_service_server::RepositoryMutationService for MutationI restored.push(path); } // Modified/Deleted: pristine content back to disk. - _ => { - let content = repo - .get_file_content(std::path::Path::new(&path)) - .map_err(repository_error)?; - if content.is_some() { + _ => match presence.get(&path) { + Some(atomic_repository::RecordedPresence::Present) => { materialize.insert(path.clone()); restored.push(path); } // No pristine content: skip (nothing safe to do). - } + Some(atomic_repository::RecordedPresence::Absent) | None => {} + Some(atomic_repository::RecordedPresence::NameConflict { + paths, + claimants, + }) => { + return Err(repository_error(atomic_repository::name_conflict_error( + &atomic_repository::tracking::normalize_path(std::path::Path::new( + &path, + )), + *paths, + *claimants, + ))); + } + }, } } if !materialize.is_empty() {