From c848c19e79ea366ad2fd93af96d2af99c1cb57b4 Mon Sep 17 00:00:00 2001 From: Bradley Hilton Date: Thu, 22 Oct 2026 13:27:44 -0500 Subject: [PATCH 1/5] fix(query): scope search, callers, and code to the current view --- Cargo.lock | 1 + README.md | 21 +- atomic-cli/src/commands/query/mod.rs | 151 ++++---- atomic-cli/src/commands/rpc.rs | 57 ++- atomic-cli/tests/query_view_filtering_test.rs | 143 +++++++ atomic-repository/Cargo.toml | 1 + atomic-repository/src/content_search.rs | 54 ++- atomic-repository/src/repository/mod.rs | 1 + atomic-repository/src/repository/query.rs | 219 +++++++++++ atomic-repository/src/repository/tests/mod.rs | 1 + .../src/repository/tests/query_view_tests.rs | 286 ++++++++++++++ .../src/repository/vault_triples.rs | 55 ++- .../knowledge/knowledge_messages.proto | 61 ++- libatomic/src/daemon/services_agent.rs | 123 +++++- libatomic/src/local.rs | 19 +- libatomic/tests/local_service_test.rs | 1 + libatomic/tests/query_view_service_test.rs | 350 ++++++++++++++++++ tests/harness/14_vault_kg.sh | 64 +++- 18 files changed, 1502 insertions(+), 106 deletions(-) create mode 100644 atomic-cli/tests/query_view_filtering_test.rs create mode 100644 atomic-repository/src/repository/query.rs create mode 100644 atomic-repository/src/repository/tests/query_view_tests.rs create mode 100644 libatomic/tests/query_view_service_test.rs diff --git a/Cargo.lock b/Cargo.lock index 561cfba3..45d57046 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,6 +453,7 @@ dependencies = [ "rayon", "redb 4.4.0", "reflink-copy", + "regex", "reqwest", "serde", "serde_json", diff --git a/README.md b/README.md index aae1a398..e1451681 100644 --- a/README.md +++ b/README.md @@ -356,19 +356,30 @@ and import edges are not stored as a parallel code graph. # Build/update the separate syntext source-content index atomic query index -# Search indexed source text; -t filters language and -g filters paths +# Search recorded source in the current view; -t filters language and -g paths atomic query code "replication" -t rs -g atomic-core/src/ # Parse a file's functions, classes, and types with line ranges atomic query entities atomic-core/src/kg/iri.rs -# Find callers by path-qualified symbol or bare name using live parsing +# Find callers by path-qualified symbol or bare name in the recorded view atomic query callers "atomic-core/src/kg/iri.rs#mint_change" --json ``` +`query search`, `query callers`, and `query code` use the **current view's effective +graph filter**: the view's changes, all ancestor views, and transitive dependencies. +They read recorded source bytes, not unrecorded working-copy edits or stale content +from another view. Deletions and renames follow the canonical view projection. +Vault-only results, such as memories and intents, remain shared. + Search combines **KG full-text search** over resource IDs, labels, and summaries -with **syntext** indexed source-content matches. The content index can be updated -with `atomic query index`; it is separate from the workflow graph. +with source-content matches verified against the recorded view. Filtering precedes +ranking and limits; no working-copy content index or checkout is required. Callers +use request-scoped tree-sitter parsing, not persisted KG code edges. This +correctness-first path renders/scans the selected view per query. RPC clients can +select a view explicitly; omitting it uses the working copy's registered desired +view. Other query verbs retain their existing scope semantics. The separate +working-copy content index is still available to its existing consumers. **SHACL validation** runs locally in the CLI using PurRDF. Builtin profiles and enterprise-authored `.atomic/shapes/*.ttl` files use the same engine: @@ -473,7 +484,7 @@ Every change stores two parallel representations: | Command | Description | |---------|-------------| | `atomic query search ` | Search the knowledge graph (structural + content) | -| `atomic query code ` | Search source code content (trigram-indexed, `-t` for file type, `-g` for path filter) | +| `atomic query code ` | Search recorded source in the current view (`-t` for file type, `-g` for path filter) | | `atomic query entities ` | List tree-sitter entities in a file (functions, classes, types with line ranges) | | `atomic query callers ` | Find callers using live source parsing, without stored KG code edges | | `atomic query neighbors ` | Explore asserted workflow relationships around a canonical resource | diff --git a/atomic-cli/src/commands/query/mod.rs b/atomic-cli/src/commands/query/mod.rs index edc627b6..01360ee8 100644 --- a/atomic-cli/src/commands/query/mod.rs +++ b/atomic-cli/src/commands/query/mod.rs @@ -22,7 +22,8 @@ pub enum QueryCommands { /// Search the knowledge graph by keywords. /// /// Performs full-text search over KG node IDs, labels, and summaries, - /// returning matching nodes ranked by relevance. + /// returning matching nodes ranked by relevance. VCS nodes and source + /// content are filtered to the current view; vault-only nodes are shared. /// /// # Examples /// @@ -49,18 +50,17 @@ pub enum QueryCommands { /// Find all functions that call a given entity. /// - /// Queries the knowledge graph for CALLS edges whose destination is the - /// specified entity node. Useful for blast-radius analysis: see every + /// Resolves calls in the current view's recorded source whose destination + /// matches a path-qualified or bare symbol. Useful for blast-radius analysis: see every /// function that breaks if this function's signature changes. /// - /// Run `atomic vault query enrich` first to populate CALLS edges. - /// Run `atomic vault query entities ` to discover entity IDs. + /// Run `atomic vault query entities ` to discover symbol names. /// /// # Examples /// /// ```text - /// atomic vault query callers entity:src/auth.rs:verify:42 - /// atomic vault query callers entity:src/auth.rs:verify:42 --json + /// atomic vault query callers 'src/auth.rs#verify' + /// atomic vault query callers verify --json /// ``` Callers(QueryCallers), @@ -80,8 +80,8 @@ pub enum QueryCommands { /// Build or update the content search index. /// /// Builds a syntext n-gram index over all source files in the repository. - /// The index is stored at `.atomic/content-index/` and is required by the - /// `code` subcommand. + /// The index is stored at `.atomic/content-index/` for working-copy content + /// consumers. View-scoped `query code` does not require it. /// /// Without `--rebuild`, performs an incremental update (fast). /// With `--rebuild`, does a full rebuild from scratch. @@ -95,10 +95,10 @@ pub enum QueryCommands { /// ``` Index(QueryIndex), - /// Search source code content using the syntext index. + /// Search recorded source code in the current view. /// - /// Searches the indexed content of all files in the repository. - /// Requires the content index to be built first via `atomic vault query enrich`. + /// Searches the view's ancestor/dependency closure, excluding sibling-view + /// and unrecorded working-copy content. No content-index rebuild is needed. /// /// # Examples /// @@ -1191,7 +1191,13 @@ impl Command for QueryNodes { } let root = find_repository_root()?; - let repo = Repository::open(&root).map_err(CliError::Repository)?; + let repo = Repository::open_readonly(&root).map_err(CliError::Repository)?; + let working_copy = repo + .require_working_copy_id() + .map_err(CliError::Repository)?; + let view = repo + .desired_view_name(working_copy) + .map_err(CliError::Repository)?; // When filtering by kind, we need enough candidates of that kind // to survive the diversity selection. Ask for a large pool so the @@ -1211,7 +1217,7 @@ impl Command for QueryNodes { }; let all_nodes = repo - .vault_kg_search(&self.query, fetch_limit, pool) + .vault_kg_search_on_view(&view, &self.query, fetch_limit, pool) .map_err(CliError::Repository)?; let nodes: Vec<_> = if let Some(ref kind_filter) = self.kind { @@ -1329,16 +1335,21 @@ pub struct QueryCallers { impl Command for QueryCallers { fn run(&self) -> CliResult<()> { - // Route every form through the service layer (the handler filters - // the subgraph to CALLS edges at the entity). + // The service resolves calls through the current view's recorded bytes. if crate::commands::rpc::query_callers(self)? { return Ok(()); } let root = find_repository_root()?; - let repo = Repository::open(&root).map_err(CliError::Repository)?; + let repo = Repository::open_readonly(&root).map_err(CliError::Repository)?; + let working_copy = repo + .require_working_copy_id() + .map_err(CliError::Repository)?; + let view = repo + .desired_view_name(working_copy) + .map_err(CliError::Repository)?; let callers = repo - .query_callers(&self.entity_id) + .query_callers_on_view(&view, &self.entity_id) .map_err(CliError::Repository)?; render_callers(&self.entity_id, &callers, self.json) } @@ -1438,7 +1449,7 @@ impl Command for QueryEntities { // QueryCode — search source code content via the syntext index // --------------------------------------------------------------------------- -/// Search source code content. +/// Search recorded source code in the current view. #[derive(Parser, Debug)] pub struct QueryCode { /// Search pattern (regex supported). @@ -1467,7 +1478,17 @@ pub struct QueryCode { impl Command for QueryCode { fn run(&self) -> CliResult<()> { + if crate::commands::rpc::query_code(self)? { + return Ok(()); + } let root = find_repository_root()?; + let repo = Repository::open_readonly(&root).map_err(CliError::Repository)?; + let working_copy = repo + .require_working_copy_id() + .map_err(CliError::Repository)?; + let view = repo + .desired_view_name(working_copy) + .map_err(CliError::Repository)?; let opts = atomic_repository::ContentSearchOptions { path_filter: self.path_filter.clone(), @@ -1477,63 +1498,57 @@ impl Command for QueryCode { case_insensitive: self.case_insensitive, }; - let result = match atomic_repository::search_content(&root, &self.pattern, opts) { - Ok(r) => r, - Err(atomic_repository::ContentSearchError::IndexNotFound) => { - println!( - "Content index not found.\n\n \ - Run `atomic vault query index` to build it, then retry.\n \ - (`atomic vault query enrich` also builds it as part of full enrichment.)" - ); - return Ok(()); - } - Err(e) => { - return Err(CliError::Internal(anyhow::anyhow!( - "code search failed: {e}" - ))); - } - }; + let result = repo + .search_content_on_view(&view, &self.pattern, opts) + .map_err(CliError::Repository)?; - if self.json { - println!("{}", serde_json::to_string_pretty(&result).unwrap()); - } else if result.matches.is_empty() { - println!("No matches found."); - } else { - // Show matches - let mut files = std::collections::HashSet::new(); - for m in &result.matches { - files.insert(m.path.as_str()); - println!( - "{}:{}: {}", - m.path, - m.line_number, - m.line_content.trim_end() - ); - } + print_code_result(&result, self.json) + } +} - // Summary +pub(crate) fn print_code_result( + result: &atomic_repository::ContentSearchResult, + json: bool, +) -> CliResult<()> { + if json { + println!("{}", serde_json::to_string_pretty(&result).unwrap()); + } else if result.matches.is_empty() { + println!("No matches found."); + } else { + // Show matches + let mut files = std::collections::HashSet::new(); + for m in &result.matches { + files.insert(m.path.as_str()); println!( - "\n{} match(es) in {} file(s).", - result.matches.len(), - files.len() + "{}:{}: {}", + m.path, + m.line_number, + m.line_content.trim_end() ); + } - // Show total and facets when results were truncated - if result.total_matches > result.matches.len() { - println!( - " ({} total matches — showing top {}. Narrow with -g or -t.)\n", - result.total_matches, - result.matches.len() - ); - println!(" Top directories:"); - for (dir, count) in &result.dir_facets { - println!(" {:<40} {} matches", format!("-g \"{}\"", dir), count); - } + // Summary + println!( + "\n{} match(es) in {} file(s).", + result.matches.len(), + files.len() + ); + + // Show total and facets when results were truncated + if result.total_matches > result.matches.len() { + println!( + " ({} total matches — showing top {}. Narrow with -g or -t.)\n", + result.total_matches, + result.matches.len() + ); + println!(" Top directories:"); + for (dir, count) in &result.dir_facets { + println!(" {:<40} {} matches", format!("-g \"{}\"", dir), count); } } - - Ok(()) } + + Ok(()) } // --------------------------------------------------------------------------- diff --git a/atomic-cli/src/commands/rpc.rs b/atomic-cli/src/commands/rpc.rs index b055f475..859cb5dd 100644 --- a/atomic-cli/src/commands/rpc.rs +++ b/atomic-cli/src/commands/rpc.rs @@ -6297,9 +6297,8 @@ pub fn query_graph(args: &super::query::QueryGraph) -> CliResult { Ok(true) } -/// `atomic query callers` over QueryGraph (Callers) — the handler filters -/// the 1-hop subgraph to CALLS edges at the entity; the CLI renders the -/// local caller lines and hints. +/// `atomic query callers` over QueryGraph (Callers) — the handler resolves +/// calls from the selected view's recorded source; the CLI renders callers. pub fn query_callers(args: &super::query::QueryCallers) -> CliResult { let Some(session) = Service::open()? else { return Ok(false); @@ -6327,6 +6326,58 @@ pub fn query_callers(args: &super::query::QueryCallers) -> CliResult { Ok(true) } +/// `query code` uses the same graph-filtered read over both local-service and +/// reactor transports. Preserve the complete domain JSON/human render. +pub fn query_code(args: &super::query::QueryCode) -> CliResult { + let Some(session) = Service::open()? else { + return Ok(false); + }; + let response = session.query_graph(pb::QueryGraphRequest { + repository: Some(session.reference.clone()), + query: Some(pb::query_graph_request::Query::ContentSearch( + pb::ContentSearchQuery { + query: args.pattern.clone(), + paths: Vec::new(), + path_filter: args.path_filter.clone(), + file_type: args.file_type.clone(), + exclude_type: None, + case_insensitive: args.case_insensitive, + max_results: Some(u32::try_from(args.max_results).map_err(|_| { + CliError::Internal(anyhow::anyhow!("max-results exceeds the RPC limit")) + })?), + }, + )), + ..Default::default() + })?; + let result = response.content_result.ok_or_else(|| { + CliError::Internal(anyhow::anyhow!( + "the service returned no content search result" + )) + })?; + let result = atomic_repository::ContentSearchResult { + matches: result + .matches + .into_iter() + .map(|m| atomic_repository::ContentMatch { + path: m.path, + line_number: m.line_number, + line_content: m.line_content, + byte_offset: m.byte_offset, + submatch_start: m.submatch_start as usize, + submatch_end: m.submatch_end as usize, + }) + .collect(), + total_matches: result.total_matches as usize, + dir_facets: result + .dir_facets + .into_iter() + .map(|facet| (facet.path, facet.count as usize)) + .collect(), + }; + super::query::print_code_result(&result, args.json)?; + Ok(true) +} + /// `atomic query plan` over QueryGraph (Plan) — the structured plan /// executes handler-side; the CLI renders the domain PlanResult JSON /// byte-for-byte (--json) or the human summary. diff --git a/atomic-cli/tests/query_view_filtering_test.rs b/atomic-cli/tests/query_view_filtering_test.rs new file mode 100644 index 00000000..a1da4df0 --- /dev/null +++ b/atomic-cli/tests/query_view_filtering_test.rs @@ -0,0 +1,143 @@ +//! CLI presentation and routing for the three view-filtered query verbs. + +use std::process::{Command, Output}; + +use atomic_core::change::ChangeHeader; +use atomic_repository::{RecordOptions, Repository, TrackingOptions}; +use serde_json::Value; + +fn run(dir: &std::path::Path, home: &std::path::Path, args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_atomic")) + .args(args) + .current_dir(dir) + .env("ATOMIC_SERVICE", "local") + .env("HOME", home) + .output() + .unwrap() +} + +fn json(dir: &std::path::Path, home: &std::path::Path, args: &[&str]) -> Value { + let output = run(dir, home, args); + assert!( + output.status.success(), + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} + +fn record(repo: &Repository, message: &str) { + repo.record( + repo.require_working_copy_id().unwrap(), + ChangeHeader::new(message), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true), + ) + .unwrap(); +} + +#[test] +fn query_verbs_render_filtered_json_and_human_output_over_the_service() { + let dir = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + let mut repo = Repository::init(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + std::fs::create_dir_all(dir.path().join("src")).unwrap(); + std::fs::write( + dir.path().join("src/lib.rs"), + "pub fn target() {}\npub fn caller() { target(); }\n", + ) + .unwrap(); + repo.add(wc, "src/lib.rs", TrackingOptions::default()) + .unwrap(); + record(&repo, "base"); + repo.kg_enrich_from_vcs(wc).unwrap(); + repo.create_draft_view("feature", "dev").unwrap(); + repo.switch_view(wc, "feature").unwrap(); + std::fs::write( + dir.path().join("src/lib.rs"), + "pub fn target(secret: SiblingSecret) {}\npub fn caller() {}\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("src/sibling.rs"), + "pub fn sibling_only() { target(); }\n", + ) + .unwrap(); + repo.add(wc, "src/sibling.rs", TrackingOptions::default()) + .unwrap(); + record(&repo, "sibling"); + repo.kg_enrich_from_vcs(wc).unwrap(); + // No content index is required by the view-scoped code/search paths. + repo.switch_view(wc, "dev").unwrap(); + drop(repo); + std::fs::write(dir.path().join("src/lib.rs"), "pub fn disk_only() {}\n").unwrap(); + + for verb in ["search", "code"] { + let result = json( + dir.path(), + home.path(), + &["query", verb, "SiblingSecret", "--json"], + ); + if verb == "search" { + assert_eq!(result, serde_json::json!([])); + } else { + assert_eq!(result["matches"], serde_json::json!([])); + assert_eq!(result["total_matches"], 0); + assert_eq!(result["dir_facets"], serde_json::json!([])); + } + } + let result = json( + dir.path(), + home.path(), + &[ + "query", + "search", + "target", + "--kind", + "https://atomic.dev/ns/workflow#File", + "--json", + ], + ); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!(result[0]["metadata"]["path"], "src/lib.rs"); + let result = json( + dir.path(), + home.path(), + &["query", "callers", "src/lib.rs#target", "--json"], + ); + assert_eq!(result.as_array().unwrap().len(), 1); + assert_eq!(result[0]["path"], "src/lib.rs"); + assert_eq!(result[0]["name"], "caller"); + assert_eq!(result[0]["line"], 2); + let result = json( + dir.path(), + home.path(), + &[ + "query", "code", "TARGET", "-i", "-g", "src/", "-t", "rs", "-n", "1", "--json", + ], + ); + assert_eq!(result["matches"].as_array().unwrap().len(), 1); + assert_eq!(result["matches"][0]["path"], "src/lib.rs"); + assert_eq!(result["matches"][0]["line_number"], 1); + assert_eq!(result["matches"][0]["byte_offset"], 7); + assert_eq!(result["total_matches"], 2); + let output = run(dir.path(), home.path(), &["query", "code", "target"]); + assert!(output.status.success()); + let text = String::from_utf8_lossy(&output.stdout); + assert!(text.contains("src/lib.rs:1: pub fn target() {}"), "{text}"); + assert!(text.contains("2 match(es) in 1 file(s)."), "{text}"); + let output = run(dir.path(), home.path(), &["query", "code", "["]); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("invalid search pattern")); + assert!(!dir + .path() + .join(".atomic/content-index/manifest.json") + .exists()); + assert_eq!( + std::fs::read_to_string(dir.path().join("src/lib.rs")).unwrap(), + "pub fn disk_only() {}\n" + ); +} diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 53cdfb1e..0fec25c4 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -45,6 +45,7 @@ ignore = { workspace = true } git2 = { workspace = true } syntext = { workspace = true } +regex = "1" tempfile = { workspace = true } log = { workspace = true } chrono = { workspace = true } diff --git a/atomic-repository/src/content_search.rs b/atomic-repository/src/content_search.rs index 60fe2fbf..0ad0ea67 100644 --- a/atomic-repository/src/content_search.rs +++ b/atomic-repository/src/content_search.rs @@ -225,6 +225,56 @@ pub fn search_content( && !ignore_rules.is_ignored(&m.path, false) }) .collect(); + Ok(rank_content_matches(raw_matches, display_limit)) +} + +/// Search graph-rendered bytes without consulting the working-copy index. +/// Path/type matching and line verification use the same syntext machinery as +/// indexed search; filtering and ranking happen before the display limit. +pub(crate) fn search_file_contents( + files: &std::collections::BTreeMap>, + pattern: &str, + opts: ContentSearchOptions, +) -> Result { + let re = regex::bytes::RegexBuilder::new(pattern) + .case_insensitive(opts.case_insensitive) + .size_limit(10 * 1024 * 1024) + .dfa_size_limit(10 * 1024 * 1024) + .build() + .map_err(|error| ContentSearchError::InvalidPattern(error.to_string()))?; + let paths: Vec<_> = files.keys().map(std::path::PathBuf::from).collect(); + let path_index = syntext::path::PathIndex::build(&paths); + let filter = syntext::path::filter::build_filter( + &path_index, + opts.file_type.as_deref(), + opts.exclude_type.as_deref(), + opts.path_filter.as_deref(), + ); + let mut matches = Vec::new(); + for (id, (path, content)) in files.iter().enumerate() { + if content.len() > 10 * 1024 * 1024 + || filter + .as_ref() + .is_some_and(|filter| !filter.file_ids.contains(id as u32)) + { + continue; + } + matches.extend(syntext::search::verifier::verify_regex( + &re, + Path::new(path), + content, + )); + } + Ok(rank_content_matches( + matches, + opts.max_results.unwrap_or(50), + )) +} + +fn rank_content_matches( + raw_matches: Vec, + display_limit: usize, +) -> ContentSearchResult { let total_matches = raw_matches.len(); // Build per-directory match counts for the facets summary. @@ -265,11 +315,11 @@ pub fn search_content( }) .collect(); - Ok(ContentSearchResult { + ContentSearchResult { matches, total_matches, dir_facets, - }) + } } /// Assign a ranking tier to a file path. diff --git a/atomic-repository/src/repository/mod.rs b/atomic-repository/src/repository/mod.rs index 9d669180..643de979 100644 --- a/atomic-repository/src/repository/mod.rs +++ b/atomic-repository/src/repository/mod.rs @@ -114,6 +114,7 @@ mod project_tree; mod projection; mod projection_commit; mod projection_effects; +mod query; pub mod ref_mapping; mod remote_cache; mod repair; diff --git a/atomic-repository/src/repository/query.rs b/atomic-repository/src/repository/query.rs new file mode 100644 index 00000000..c7bf9859 --- /dev/null +++ b/atomic-repository/src/repository/query.rs @@ -0,0 +1,219 @@ +//! Read-only queries over recorded view projections, not ambient KG or disk bytes. + +use std::collections::{BTreeMap, HashSet}; + +use atomic_core::kg::{ + iri, + ontology::{Class, Predicate}, +}; +use atomic_core::pristine::KgTxnT; +use atomic_core::KgNode; + +use super::*; +use crate::content_search::{search_file_contents, ContentSearchOptions, ContentSearchResult}; + +pub(super) struct QueryView { + pub files: BTreeMap>, + changes: HashSet, + views: HashSet, + nodes: BTreeMap, +} + +impl QueryView { + pub fn nodes(&self) -> &BTreeMap { + &self.nodes + } + + /// Project ambient candidates before ranking. File identities are canonical, + /// but their presence must come from the selected view's current projection. + pub fn project_node( + &self, + txn: &T, + node: KgNode, + ) -> Result, RepositoryError> { + let visible = if node.has_type(Class::File) { + return Ok(self.nodes.get(&node.id).cloned()); + } else if node.has_type(Class::Change) { + self.changes.contains(&node.id) + } else if node.has_type(Class::View) { + self.views.contains(&node.id) + } else if node.has_type(Class::Identity) && node.source == "vcs" { + let mut visible = false; + for edge in txn.get_kg_edges_to(&node.id)? { + if edge.kind != Predicate::AuthoredBy { + continue; + } + let Some(authoring_node) = txn.get_kg_node(&edge.from_id)? else { + continue; + }; + // An identity shared with a vault record stays shared. A VCS-only + // identity needs an authored change visible in the selected view. + if !authoring_node.has_type(Class::Change) + || self.project_node(txn, authoring_node)?.is_some() + { + visible = true; + break; + } + } + visible + } else { + // Vault and session resources have no VCS view membership. + true + }; + Ok(visible.then_some(node)) + } + + pub fn search_content( + &self, + pattern: &str, + opts: ContentSearchOptions, + ) -> Result { + search_file_contents(&self.files, pattern, opts).map_err(|error| { + RepositoryError::InvalidOperation { + message: format!("code search failed: {error}"), + } + }) + } +} + +impl Repository { + pub(super) fn query_view(&self, view_name: &str) -> Result { + let txn = self.pristine.read_txn()?; + let view = txn + .get_view(view_name)? + .ok_or_else(|| RepositoryError::ViewNotFound { + name: view_name.to_string(), + })?; + let visibility = graph_visibility_closure(&txn, &view)?; + let projection = self.project_tree_for_visibility(&txn, &visibility)?; + if !projection.name_conflicts.is_empty() { + return Err(RepositoryError::InvalidOperation { + message: "cannot query a view with unresolved path claims".to_string(), + }); + } + let mut changes = HashSet::new(); + for id in visibility + .iter_dependency_first() + .filter(|id| !id.is_root()) + { + let hash = txn.get_external(*id)?.ok_or_else(|| { + RepositoryError::Database(format!("visible change {id} has no hash")) + })?; + changes.insert(iri::mint_change(&hash)); + } + let project = self.project_id()?; + let views = txn + .resolve_full_view_chain(&view)? + .into_iter() + .map(|view| iri::mint_view(&project, &view.name)) + .collect(); + let rules = IgnoreRules::load_for_enrichment(self.root()); + let mut files = BTreeMap::new(); + for (path, item) in projection.present { + if item.is_directory + || crate::ignore::is_enrichment_internal(Path::new(&path)) + || rules.is_ignored(Path::new(&path), false) + { + continue; + } + let attrs = atomic_core::output::project_inode_attributes( + &txn, + item.position, + visibility.attribute_visibility(), + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + if attrs.is_conflicted() { + return Err(RepositoryError::InvalidOperation { + message: format!("cannot query '{path}' with conflicting inode attributes"), + }); + } + // Never follow a symlink into working-copy or external content. + if attrs.materialization.kind != atomic_core::change::InodeKind::Regular { + continue; + } + let bytes = super::content::retrieve_content_with_filter_fast( + &txn, + &self.change_store, + item.inode, + item.position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(visibility.clone()), + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + files.insert(path, bytes); + } + let nodes = files + .keys() + .map(|path| { + let id = iri::mint_file(&project, path); + let label = path.rsplit('/').next().unwrap_or(path); + let node = KgNode::new(&id, Class::File, label, "vcs") + .with_metadata(serde_json::json!({"path": path})); + (id, node) + }) + .collect(); + Ok(QueryView { + files, + changes, + views, + nodes, + }) + } + + /// Search recorded source bytes through a view's validated ancestor and + /// dependency closure. Unrecorded files/edits and the working-copy content + /// index are excluded; no index rebuild or checkout occurs. + pub fn search_content_on_view( + &self, + view_name: &str, + pattern: &str, + opts: ContentSearchOptions, + ) -> Result { + self.search_content_paths_on_view(view_name, pattern, opts, &[]) + } + + /// Like [`Self::search_content_on_view`], restricted to a union of exact + /// repository-relative paths and directory prefixes before matching. + pub fn search_content_paths_on_view( + &self, + view_name: &str, + pattern: &str, + opts: ContentSearchOptions, + paths: &[String], + ) -> Result { + let mut scope = self.query_view(view_name)?; + if !paths.is_empty() { + scope.files.retain(|path, _| { + paths.iter().any(|prefix| { + let prefix = prefix.trim_end_matches('/'); + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|rest| rest.starts_with('/')) + }) + }); + } + scope.search_content(pattern, opts) + } + + /// Resolve callers from the selected view's recorded source using the same + /// request-scoped parser as working-copy caller queries. No KG code edges + /// are created or required. Accepts `#` or a bare symbol. + pub fn query_callers_on_view( + &self, + view_name: &str, + target: &str, + ) -> Result, RepositoryError> { + let scope = self.query_view(view_name)?; + let sources: Vec<_> = scope + .files + .iter() + .filter_map(|(path, bytes)| { + std::str::from_utf8(bytes) + .ok() + .map(|source| (path.clone(), source.to_owned())) + }) + .collect(); + Ok(SyntaxIndex::from_sources(&sources).callers(target)) + } +} diff --git a/atomic-repository/src/repository/tests/mod.rs b/atomic-repository/src/repository/tests/mod.rs index f569e581..4c744e1a 100644 --- a/atomic-repository/src/repository/tests/mod.rs +++ b/atomic-repository/src/repository/tests/mod.rs @@ -37,6 +37,7 @@ mod operation_routing_tests; mod operation_undo_tests; mod path_claim_migration_tests; mod projection_effects_tests; +mod query_view_tests; mod record_duplication_tests; mod record_tests; mod ref_mapping_tests; diff --git a/atomic-repository/src/repository/tests/query_view_tests.rs b/atomic-repository/src/repository/tests/query_view_tests.rs new file mode 100644 index 00000000..49f7530d --- /dev/null +++ b/atomic-repository/src/repository/tests/query_view_tests.rs @@ -0,0 +1,286 @@ +use super::*; +use crate::{ContentSearchOptions, RecordOptions}; +use atomic_core::kg::{iri, ontology::Class}; +use atomic_core::pristine::vault::KgNode; +use atomic_core::pristine::{KgMutTxnT, ViewTxnT}; + +fn record(repo: &TestRepository, message: &str) -> Hash { + let author = if message == "sibling_history_marker" { + "HiddenSiblingAuthor" + } else { + "VisibleBaseAuthor" + }; + *repo + .record( + ChangeHeader::builder() + .message(message) + .author(Author::new(author, None::)) + .build(), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true), + ) + .unwrap() + .hash() +} + +fn fixture() -> (TempDir, TestRepository, Hash) { + let (temp, mut repo) = create_temp_repo(); + std::fs::create_dir_all(temp.path().join("src")).unwrap(); + std::fs::write( + temp.path().join("src/lib.rs"), + "pub fn target() {}\npub fn caller() { target(); }\n// inherited_marker\n", + ) + .unwrap(); + repo.add("src/lib.rs", TrackingOptions::default()).unwrap(); + record(&repo, "inherited history"); + repo.kg_enrich_from_vcs(repo.working_copy()).unwrap(); + repo.create_draft_view("feature", "dev").unwrap(); + repo.switch_view("feature").unwrap(); + std::fs::write( + temp.path().join("src/lib.rs"), + "pub fn target(secret: SiblingSecret) {}\npub fn caller() {}\n// inherited_marker\n", + ) + .unwrap(); + std::fs::write( + temp.path().join("src/sibling.rs"), + "pub fn sibling_only() { target(); }\n// sibling_marker\n", + ) + .unwrap(); + repo.add("src/sibling.rs", TrackingOptions::default()) + .unwrap(); + let feature = record(&repo, "sibling_history_marker"); + repo.kg_enrich_from_vcs(repo.working_copy()).unwrap(); + crate::build_content_index(temp.path()).unwrap(); + (temp, repo, feature) +} + +#[test] +fn query_search_filters_before_ranking_and_uses_recorded_sources() { + let (_temp, repo, _) = fixture(); + for query in [ + "sibling", + "SiblingSecret", + "sibling_history_marker", + "HiddenSiblingAuthor", + ] { + let nodes = repo + .vault_kg_search_on_view("dev", query, 1, Some(1)) + .unwrap(); + assert!(nodes.is_empty(), "{query}: {nodes:?}"); + assert!(!repo + .vault_kg_search_on_view("feature", query, 20, Some(50)) + .unwrap() + .is_empty()); + } + let target = repo + .query_callers_on_view("dev", "src/lib.rs#target") + .unwrap() + .into_iter() + .find(|caller| caller.name == "caller") + .unwrap(); + assert_eq!(target.summary.as_deref(), Some("pub fn caller()")); + assert!(!repo + .vault_kg_search_on_view("dev", "VisibleBaseAuthor", 1, None) + .unwrap() + .is_empty()); + // A low result limit is applied only after filtering hidden candidates. + assert!(!repo + .vault_kg_search_on_view("dev", "inherited", 1, Some(1)) + .unwrap() + .is_empty()); + let memory_id = iri::mint_memory(&repo.project_id().unwrap(), "shared"); + let mut txn = repo.pristine().write_txn().unwrap(); + txn.upsert_kg_node(&KgNode::new( + &memory_id, + Class::Memory, + "universal_marker", + "vault", + )) + .unwrap(); + txn.commit().unwrap(); + assert_eq!( + repo.vault_kg_search_on_view("dev", "universal_marker", 1, None) + .unwrap()[0] + .id, + memory_id + ); +} + +#[test] +fn query_callers_uses_recorded_calls_not_stale_shared_edges_or_disk() { + let (temp, repo, _) = fixture(); + let target = "src/lib.rs#target"; + let callers = |view: &str| { + repo.query_callers_on_view(view, target) + .unwrap() + .into_iter() + .map(|caller| format!("{}#{}:{}", caller.path, caller.name, caller.line)) + .collect::>() + }; + assert_eq!(callers("dev"), ["src/lib.rs#caller:2"]); + assert_eq!(callers("feature"), ["src/sibling.rs#sibling_only:1"]); + std::fs::write(temp.path().join("src/lib.rs"), "pub fn disk_only() {}\n").unwrap(); + assert_eq!(callers("dev"), ["src/lib.rs#caller:2"]); + assert!(repo + .query_callers_on_view("dev", "src/sibling.rs#sibling_only") + .unwrap() + .is_empty()); +} + +#[test] +fn query_code_ignores_stale_indexes_and_filters_counts_facets_and_limits() { + let (temp, repo, _) = fixture(); + let opts = ContentSearchOptions { + max_results: Some(1), + file_type: Some("rs".into()), + path_filter: Some("src/".into()), + case_insensitive: true, + ..Default::default() + }; + let result = repo + .search_content_on_view("dev", "TARGET|INHERITED_MARKER", opts.clone()) + .unwrap(); + assert_eq!(result.matches.len(), 1); + assert_eq!(result.total_matches, 3); + assert_eq!(result.dir_facets, [("src/lib.rs".into(), 3)]); + assert_eq!(result.matches[0].path, "src/lib.rs"); + assert_eq!(result.matches[0].line_number, 1); + assert_eq!(result.matches[0].byte_offset, 7); + assert_eq!(result.matches[0].submatch_start, 7); + assert_eq!(result.matches[0].submatch_end, 13); + for query in ["SiblingSecret", "sibling_marker", "disk_only"] { + std::fs::write(temp.path().join("src/lib.rs"), "pub fn disk_only() {}\n").unwrap(); + let result = repo + .search_content_on_view("dev", query, opts.clone()) + .unwrap(); + assert_eq!(result.total_matches, 0, "{query}"); + assert!(result.matches.is_empty()); + assert!(result.dir_facets.is_empty()); + } + let result = repo + .search_content_on_view("feature", "sibling_marker", opts) + .unwrap(); + assert_eq!(result.matches[0].path, "src/sibling.rs"); + assert!(repo + .search_content_on_view("dev", "[", ContentSearchOptions::default()) + .is_err()); +} + +#[test] +fn queries_include_ancestors_and_transitive_dependencies_without_materializing() { + let (temp, mut repo, feature_hash) = fixture(); + repo.create_draft_view("child", "feature").unwrap(); + repo.create_draft_view("grandchild", "child").unwrap(); + repo.create_view_with_identity("empty", ViewScope::Shared, None) + .unwrap(); + assert!(repo + .vault_kg_search_on_view("empty", "target", 10, None) + .unwrap() + .is_empty()); + assert_eq!( + repo.search_content_on_view("empty", "target", ContentSearchOptions::default()) + .unwrap() + .total_matches, + 0 + ); + assert!(repo + .query_callers_on_view("empty", "src/lib.rs#target") + .unwrap() + .is_empty()); + repo.create_view_with_identity("tip-only", ViewScope::Shared, None) + .unwrap(); + // Reference only the tip: its required base must be included by the + // canonical dependency closure, not by direct membership or parent names. + let mut txn = repo.pristine().write_txn().unwrap(); + let mut view = txn.get_view("tip-only").unwrap().unwrap(); + let id = txn.get_internal(&feature_hash).unwrap().unwrap(); + txn.put_change(&mut view, id, &feature_hash).unwrap(); + txn.update_view(&view).unwrap(); + txn.commit().unwrap(); + let before = std::fs::read(temp.path().join("src/lib.rs")).unwrap(); + for view in ["child", "grandchild", "tip-only"] { + assert!(!repo + .vault_kg_search_on_view(view, "inherited", 10, None) + .unwrap() + .is_empty()); + assert_eq!( + repo.search_content_on_view(view, "inherited_marker", ContentSearchOptions::default()) + .unwrap() + .total_matches, + 1 + ); + assert!(!repo + .query_callers_on_view(view, "src/lib.rs#target") + .unwrap() + .is_empty()); + } + assert_eq!( + std::fs::read(temp.path().join("src/lib.rs")).unwrap(), + before + ); + assert_eq!( + repo.desired_view_name(repo.working_copy()).unwrap(), + "feature" + ); + assert!(repo + .vault_kg_search_on_view("missing", "target", 10, None) + .is_err()); +} + +#[test] +fn queries_exclude_deleted_and_renamed_paths_without_an_index_rebuild() { + let (temp, repo) = create_temp_repo(); + std::fs::write(temp.path().join("old.rs"), "pub fn renamed_marker() {}\n").unwrap(); + std::fs::write( + temp.path().join("deleted.rs"), + "pub fn deleted_marker() {}\n", + ) + .unwrap(); + repo.add("old.rs", TrackingOptions::default()).unwrap(); + repo.add("deleted.rs", TrackingOptions::default()).unwrap(); + record(&repo, "before lifecycle changes"); + repo.kg_enrich_from_vcs(repo.working_copy()).unwrap(); + crate::build_content_index(temp.path()).unwrap(); + std::fs::rename(temp.path().join("old.rs"), temp.path().join("renamed.rs")).unwrap(); + std::fs::remove_file(temp.path().join("deleted.rs")).unwrap(); + record(&repo, "after lifecycle changes"); + let matches = repo + .search_content_on_view("dev", "marker", ContentSearchOptions::default()) + .unwrap(); + assert_eq!(matches.total_matches, 1); + assert_eq!(matches.matches[0].path, "renamed.rs"); + let nodes = repo + .vault_kg_search_on_view("dev", "marker", 50, None) + .unwrap(); + assert!(nodes + .iter() + .all(|node| !node.id.contains("old.rs") && !node.id.contains("deleted.rs"))); + assert!(repo + .vault_kg_search_on_view("dev", "renamed.rs", 20, None) + .unwrap() + .iter() + .any(|node| node.id == iri::mint_file(&repo.project_id().unwrap(), "renamed.rs"))); +} + +#[test] +fn invalid_dependency_closure_fails_closed_for_every_query() { + let (_temp, repo, hash) = fixture(); + let missing = Hash::of(b"unregistered query dependency"); + let mut txn = repo.pristine().write_txn().unwrap(); + let id = txn.get_internal(&hash).unwrap().unwrap(); + txn.put_change_deps(id, &[missing]).unwrap(); + txn.commit().unwrap(); + let errors = [ + repo.vault_kg_search_on_view("feature", "sibling", 10, None) + .unwrap_err(), + repo.search_content_on_view("feature", "sibling", ContentSearchOptions::default()) + .unwrap_err(), + repo.query_callers_on_view("feature", "src/lib.rs#target") + .unwrap_err(), + ]; + for error in errors { + assert!(error.to_string().contains(&missing.to_base32()), "{error}"); + } +} diff --git a/atomic-repository/src/repository/vault_triples.rs b/atomic-repository/src/repository/vault_triples.rs index 20697ed0..3f3a267f 100644 --- a/atomic-repository/src/repository/vault_triples.rs +++ b/atomic-repository/src/repository/vault_triples.rs @@ -239,11 +239,47 @@ impl Repository { query: &str, limit: usize, pool: Option, + ) -> Result, RepositoryError> { + self.vault_kg_search_inner(query, limit, pool, None) + } + + /// Search VCS nodes and source content through the requested view's + /// validated closure. Filtering precedes candidate ranking and truncation. + /// Vault-only nodes remain repository-wide. + pub fn vault_kg_search_on_view( + &self, + view_name: &str, + query: &str, + limit: usize, + pool: Option, + ) -> Result, RepositoryError> { + let scope = self.query_view(view_name)?; + self.vault_kg_search_inner(query, limit, pool, Some(&scope)) + } + + fn vault_kg_search_inner( + &self, + query: &str, + limit: usize, + pool: Option, + scope: Option<&super::query::QueryView>, ) -> Result, RepositoryError> { let txn = self.pristine.read_txn()?; let mut rows = BTreeMap::new(); let mut content = HashMap::::new(); - if crate::content_search::has_content_index(self.root()) { + if let Some(scope) = scope { + if let Ok(result) = scope.search_content( + query, + crate::content_search::ContentSearchOptions { + max_results: Some(2000), + ..Default::default() + }, + ) { + for result in result.matches { + *content.entry(result.path).or_default() += 1; + } + } + } else if crate::content_search::has_content_index(self.root()) { if let Ok(result) = crate::content_search::search_content( self.root(), query, @@ -259,12 +295,29 @@ impl Repository { } for (id, _) in txn.kg_fts_match_ids(query)? { if let Some(node) = txn.get_kg_node(&id)? { + let node = if let Some(scope) = scope { + let Some(node) = scope.project_node(&txn, node)? else { + continue; + }; + node + } else { + node + }; let hits = fts_hits(&node, query); if hits > 0 { rows.insert(id, (node, hits)); } } } + // A visible rename or new file need not already have a KG posting. + if let Some(scope) = scope { + for node in scope.nodes().values() { + let hits = fts_hits(node, query); + if hits > 0 { + rows.insert(node.id.clone(), (node.clone(), hits)); + } + } + } let project = self.project_id()?; for path in content.keys() { let id = iri::mint_file(&project, path); diff --git a/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto b/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto index be98ca27..780c9965 100644 --- a/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto +++ b/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto @@ -8,7 +8,8 @@ // GetProjectIdentity and GetWorkflowSnapshot provide read-only, repository-wide // workflow inspection for `repo identity show` and client-side SHACL validation. // Local file-index construction (`query index`) remains client-side. QueryGraph -// reads a view-scoped indexed snapshot; it must never build/refresh an index. +// reads are read-only; the view-scoped search/callers/content paths project +// recorded graph bytes and must never build/refresh a persistent index. syntax = "proto3"; @@ -45,7 +46,7 @@ message QueryGraphRequest { RepositoryRef repository = 1; uint32 limit = 2; // default 10-20 per kind today oneof query { - ContentSearchQuery content_search = 3; // `query code` (syntext index) + ContentSearchQuery content_search = 3; // `query code` (recorded view bytes) KgSearchQuery kg_search = 4; // `query search` EntityQuery entity = 5; // `query entities` NeighborsQuery neighbors = 6; // `query neighbors` @@ -54,21 +55,32 @@ message QueryGraphRequest { PlanExecutionQuery plan = 9; // `query plan` (JSON plan steps) RagAskQuery rag_ask = 10; // `query ask` (KG + LLM answer) } - ViewRef view = 11; // required; sandbox must match grant - // All nodes, edges, snippets, opaque payloads and every plan step are filtered - // BEFORE evaluation/provider calls by this effective perspective. Plans are - // read-only; no embed/enrich/reindex or arbitrary filesystem/provider action. + // For kg_search, callers, and content_search: use this view's full ancestor + // and dependency closure. Local callers may omit it to use WORKING_COPIES' + // authoritative desired view, never the current_view compatibility file. + // An explicit name/id mismatch or unknown view is an error, not a fallback. + // VCS candidates are filtered BEFORE ranking/limits. Vault-only KG records + // remain repository-wide. These reads never refresh indexes or materialize. + // Other query kinds retain their existing scope semantics. + ViewRef view = 11; } message ContentSearchQuery { string query = 1; + // Optional union of repository-relative paths or directory prefixes. repeated string paths = 2; - // SHAPE-CONFIRM: filters from query/mod.rs content search. + optional string path_filter = 3; // same glob semantics as `query code -g` + optional string file_type = 4; + optional string exclude_type = 5; + bool case_insensitive = 6; + // Omitted: use QueryGraphRequest.limit (or 30 when it is zero). + // Explicit zero: no displayed matches; totals/facets are still returned. + optional uint32 max_results = 7; } message KgSearchQuery { string query = 1; - // `query search --kind` (add-only): keep only nodes of this kind - // (case-insensitive), applied over the candidate pool before the limit. + // `query search --kind` (add-only): keep only nodes asserting this absolute + // class IRI, applied over the candidate pool before the limit. optional string kind = 2; // The candidate pool size (add-only): how many candidates to score // before selecting the top results (the CLI's --pool, default 5000). @@ -84,7 +96,9 @@ message NeighborsQuery { optional uint32 depth = 3; } message CallersQuery { - string node_id = 1; // `#` or a bare symbol, from live source + // `#` or a bare symbol, resolved from the selected view's + // recorded source. Working-copy edits and sibling views are excluded. + string node_id = 1; } message CodeCaller { string path = 1; @@ -129,8 +143,8 @@ message QueryGraphResponse { // The pre-cap node count (add-only): the `query graph` cap note's "N // nodes collected" figure. optional uint32 graph_collected = 7; - // Whether the neighbors subgraph holds ANY calls edge (add-only): the - // `query callers` hint's entry-point distinction. + // Whether the target has any incoming or outgoing call in the selected + // view. Retained for compatibility; no enrichment is required by callers. bool calls_present = 8; // The full agentic ask result (add-only; schema "atomic.query.ask.v1" = // the domain AgentResult JSON) plus its elapsed time. @@ -138,6 +152,29 @@ message QueryGraphResponse { optional uint64 ask_elapsed_ms = 10; uint32 api_version = 11; // canonical workflow KG API version 1 repeated CodeCaller callers = 12; + // Complete typed `query code` result, including scoped totals, facets and + // offsets. `hits` also carries the displayed lines for generic clients. + ContentSearchResult content_result = 13; +} + +message ContentSearchResult { + repeated ContentMatch matches = 1; + uint64 total_matches = 2; + repeated ContentDirectoryFacet dir_facets = 3; +} + +message ContentMatch { + string path = 1; + uint32 line_number = 2; + string line_content = 3; + uint64 byte_offset = 4; + uint64 submatch_start = 5; + uint64 submatch_end = 6; +} + +message ContentDirectoryFacet { + string path = 1; + uint64 count = 2; } message KGNode { diff --git a/libatomic/src/daemon/services_agent.rs b/libatomic/src/daemon/services_agent.rs index 1ec4275f..1df33054 100644 --- a/libatomic/src/daemon/services_agent.rs +++ b/libatomic/src/daemon/services_agent.rs @@ -3343,6 +3343,42 @@ pub struct KnowledgeImpl { pub state: Arc, } +fn query_view_name( + repo: &atomic_repository::Repository, + requested: Option<&ViewRef>, +) -> Result { + let Some(requested) = requested else { + let working_copy = repo.require_working_copy_id().map_err(repository_error)?; + return repo + .desired_view_name(working_copy) + .map_err(repository_error); + }; + let name = match requested.name.as_deref().filter(|name| !name.is_empty()) { + Some(name) => name.to_string(), + None if !requested.view_id.is_empty() => repo + .list_views() + .map_err(repository_error)? + .into_iter() + .find(|name| super::sandbox_grants::view_ref_id(name) == requested.view_id) + .ok_or_else(|| domain_status(ErrorCode::View, "unknown query view ID"))?, + None => { + return Err(domain_status( + ErrorCode::InvalidArgument, + "query view must name a view or ID", + )) + } + }; + if !requested.view_id.is_empty() + && requested.view_id != super::sandbox_grants::view_ref_id(&name) + { + return Err(domain_status( + ErrorCode::InvalidArgument, + "query view name and ID disagree", + )); + } + Ok(name) +} + #[tonic::async_trait] impl knowledge_service_server::KnowledgeService for KnowledgeImpl { async fn get_project_identity( @@ -3449,9 +3485,76 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { request.limit as usize }; let root = handle.root.clone(); + let requested_view = request.view; + let content_limit = if request.limit == 0 { + 30 + } else { + request.limit as usize + }; let response = tokio::task::spawn_blocking(move || match query { + QueryOneof::ContentSearch(search) => { + let repo = handle.repository_readonly()?; + let view = query_view_name(&repo, requested_view.as_ref())?; + let result = repo + .search_content_paths_on_view( + &view, + &search.query, + atomic_repository::ContentSearchOptions { + path_filter: search.path_filter, + file_type: search.file_type, + exclude_type: search.exclude_type, + max_results: Some( + search + .max_results + .map(|n| n as usize) + .unwrap_or(content_limit), + ), + case_insensitive: search.case_insensitive, + }, + &search.paths, + ) + .map_err(repository_error)?; + Ok(QueryGraphResponse { + api_version: 1, + hits: result + .matches + .iter() + .map(|m| ContentHit { + path: m.path.clone(), + snippet: m.line_content.clone(), + line: Some(m.line_number as u64), + score: None, + }) + .collect(), + content_result: Some(ContentSearchResult { + matches: result + .matches + .into_iter() + .map(|m| ContentMatch { + path: m.path, + line_number: m.line_number, + line_content: m.line_content, + byte_offset: m.byte_offset, + submatch_start: m.submatch_start as u64, + submatch_end: m.submatch_end as u64, + }) + .collect(), + total_matches: result.total_matches as u64, + dir_facets: result + .dir_facets + .into_iter() + .map(|(path, count)| ContentDirectoryFacet { + path, + count: count as u64, + }) + .collect(), + }), + ..Default::default() + }) + } QueryOneof::KgSearch(search) => { - let repo = handle.repository()?; + let repo = handle.repository_readonly()?; + let view = query_view_name(&repo, requested_view.as_ref())?; // The candidate pool the CLI's --pool names (default 5000) // — the pool is what lets lower-scored kinds survive the // diversity selection, so it rides the request. @@ -3463,7 +3566,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { // kind's nodes, take the limit AFTER filtering — the // local post-filter order. let kind = kind_filter; - repo.vault_kg_search(&search.query, pool, Some(pool)) + repo.vault_kg_search_on_view(&view, &search.query, pool, Some(pool)) .map_err(repository_error)? .into_iter() .filter(|node| node.types.contains(&kind)) @@ -3471,7 +3574,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { .map(|node| crate::daemon::services_query::kg_node_proto(node.clone())) .collect::>() } else { - repo.vault_kg_search(&search.query, limit, Some(pool)) + repo.vault_kg_search_on_view(&view, &search.query, limit, Some(pool)) .map_err(repository_error)? .into_iter() .map(|node| crate::daemon::services_query::kg_node_proto(node.clone())) @@ -3490,6 +3593,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }) } QueryOneof::Neighbors(neighbors) => { @@ -3512,6 +3616,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }) } QueryOneof::Entity(entity) => { @@ -3551,12 +3656,14 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }) } QueryOneof::Callers(callers) => { let repo = handle.repository_readonly()?; + let view = query_view_name(&repo, requested_view.as_ref())?; let callers = repo - .query_callers(&callers.node_id) + .query_callers_on_view(&view, &callers.node_id) .map_err(repository_error)? .into_iter() .map(|caller| crate::atomic::CodeCaller { @@ -3626,6 +3733,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }); } // 2. Expand: per-seed neighbor subgraphs, kind-filtered, @@ -3738,6 +3846,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }) } QueryOneof::Plan(plan) => { @@ -3765,6 +3874,7 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { calls_present: false, ask_result: None, ask_elapsed_ms: None, + content_result: None, }) } QueryOneof::RagAsk(ask) => { @@ -3812,12 +3922,9 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { payload, }), ask_elapsed_ms: Some(elapsed.as_millis() as u64), + content_result: None, }) } - other => Err(domain_status( - ErrorCode::InvalidArgument, - format!("query kind {other:?} lands with its slice"), - )), }) .await .map_err(|e| Status::internal(e.to_string()))??; diff --git a/libatomic/src/local.rs b/libatomic/src/local.rs index 9601b138..17861709 100644 --- a/libatomic/src/local.rs +++ b/libatomic/src/local.rs @@ -11,16 +11,17 @@ use tonic::{Request, Status}; use crate::atomic::{ attestation_service_server::AttestationService, daemon_service_server::DaemonService, + knowledge_service_server::KnowledgeService, repository_mutation_service_server::RepositoryMutationService, repository_query_service_server::RepositoryQueryService, vault_service_server::VaultService, view_service_server::ViewService, AddFilesRequest, CreateViewRequest, DeleteViewRequest, GetChangeRequest, GetVaultContextRequest, GetVaultEntryRequest, ListVaultEntriesRequest, - ListViewsRequest, LogRequest, PreviewRestoreRequest, RecordRequest, ResolveRepositoryRequest, - StatusRequest, SwitchViewRequest, VerifyAttestationRequest, + ListViewsRequest, LogRequest, PreviewRestoreRequest, QueryGraphRequest, RecordRequest, + ResolveRepositoryRequest, StatusRequest, SwitchViewRequest, VerifyAttestationRequest, }; use crate::daemon::{ services::{DaemonImpl, MutationImpl, QueryImpl}, - services_agent::{AttestationImpl, VaultImpl}, + services_agent::{AttestationImpl, KnowledgeImpl, VaultImpl}, services_query::ViewImpl, state::DaemonState, }; @@ -33,7 +34,7 @@ use crate::daemon::{ /// /// Supports repository resolution, basic working-copy/change/view operations, /// read-only restore previews (including recorded file bytes), -/// read-only vault get/list/context, and attestation verification. View-scoped +/// read-only knowledge queries, vault get/list/context, and attestation verification. View-scoped /// vault reads are not supported; unsupported methods fail explicitly with /// `Unimplemented`. #[derive(Clone)] @@ -233,6 +234,16 @@ impl LocalService { .await?; Ok(response.into_inner().encode_to_vec()) } + "/atomic.KnowledgeService/QueryGraph" => { + let request = decode::(request)?; + require_repository(&request.repository)?; + let response = KnowledgeImpl { + state: self.state.clone(), + } + .query_graph(Request::new(request)) + .await?; + Ok(response.into_inner().encode_to_vec()) + } "/atomic.AttestationService/VerifyAttestation" => { let request = decode::(request)?; require_repository(&request.repository)?; diff --git a/libatomic/tests/local_service_test.rs b/libatomic/tests/local_service_test.rs index 9bd384c3..e4173a01 100644 --- a/libatomic/tests/local_service_test.rs +++ b/libatomic/tests/local_service_test.rs @@ -88,6 +88,7 @@ async fn rejects_invalid_requests_and_unknown_methods() { "/atomic.ViewService/CreateView", "/atomic.ViewService/SwitchView", "/atomic.ViewService/DeleteView", + "/atomic.KnowledgeService/QueryGraph", ] { let error = service.call(method, &[]).await.unwrap_err(); assert_eq!(error.code(), tonic::Code::InvalidArgument, "{method}"); diff --git a/libatomic/tests/query_view_service_test.rs b/libatomic/tests/query_view_service_test.rs new file mode 100644 index 00000000..5909310b --- /dev/null +++ b/libatomic/tests/query_view_service_test.rs @@ -0,0 +1,350 @@ +//! Knowledge queries over the actual protobuf dispatch boundary shared by +//! LocalService and the daemon, including explicit non-materialized views. + +use atomic_core::change::ChangeHeader; +use atomic_repository::{RecordOptions, Repository, TrackingOptions}; +use libatomic::{proto as pb, LocalService}; +use prost::Message; + +fn record(repo: &Repository, message: &str) { + repo.record( + repo.require_working_copy_id().unwrap(), + ChangeHeader::new(message), + RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true), + ) + .unwrap(); +} + +async fn fixture() -> (tempfile::TempDir, LocalService, pb::RepositoryRef) { + let dir = tempfile::tempdir().unwrap(); + let mut repo = Repository::init(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + std::fs::create_dir_all(dir.path().join("src")).unwrap(); + std::fs::write( + dir.path().join("src/lib.rs"), + "pub fn target() {}\npub fn caller() { target(); }\n// inherited_marker\n", + ) + .unwrap(); + repo.add(wc, "src/lib.rs", TrackingOptions::default()) + .unwrap(); + record(&repo, "inherited history"); + repo.kg_enrich_from_vcs(wc).unwrap(); + repo.create_draft_view("feature", "dev").unwrap(); + repo.switch_view(wc, "feature").unwrap(); + std::fs::write( + dir.path().join("src/lib.rs"), + "pub fn target(secret: SiblingSecret) {}\npub fn caller() {}\n// inherited_marker\n", + ) + .unwrap(); + std::fs::write( + dir.path().join("src/hidden.rs"), + "pub fn sibling_only() { target(); }\n// sibling_marker\n", + ) + .unwrap(); + repo.add(wc, "src/hidden.rs", TrackingOptions::default()) + .unwrap(); + record(&repo, "sibling history"); + repo.kg_enrich_from_vcs(wc).unwrap(); + atomic_repository::build_content_index(dir.path()).unwrap(); + repo.switch_view(wc, "dev").unwrap(); + drop(repo); + // Neither this derived marker nor unrecorded disk bytes are authoritative. + std::fs::write(dir.path().join(".atomic/current_view"), "feature").unwrap(); + std::fs::write(dir.path().join("src/lib.rs"), "pub fn disk_only() {}\n").unwrap(); + let service = LocalService::new(); + let bytes = service + .call( + "/atomic.DaemonService/ResolveRepository", + &pb::ResolveRepositoryRequest { + path: dir.path().display().to_string(), + } + .encode_to_vec(), + ) + .await + .unwrap(); + let reference = pb::ResolveRepositoryResponse::decode(bytes.as_slice()) + .unwrap() + .repository + .unwrap(); + (dir, service, reference) +} + +fn view(name: &str) -> pb::ViewRef { + pb::ViewRef { + name: Some(name.into()), + ..Default::default() + } +} + +async fn query( + service: &LocalService, + repository: &pb::RepositoryRef, + view: Option, + query: pb::query_graph_request::Query, +) -> Result { + let bytes = service + .call( + "/atomic.KnowledgeService/QueryGraph", + &pb::QueryGraphRequest { + repository: Some(repository.clone()), + view, + query: Some(query), + limit: 1, + } + .encode_to_vec(), + ) + .await?; + Ok(pb::QueryGraphResponse::decode(bytes.as_slice()).unwrap()) +} + +fn search(term: &str) -> pb::query_graph_request::Query { + pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: term.into(), + pool: Some(1), + ..Default::default() + }) +} + +fn code(term: &str) -> pb::query_graph_request::Query { + pb::query_graph_request::Query::ContentSearch(pb::ContentSearchQuery { + query: term.into(), + ..Default::default() + }) +} + +fn callers() -> pb::query_graph_request::Query { + pb::query_graph_request::Query::Callers(pb::CallersQuery { + node_id: "src/lib.rs#target".into(), + }) +} + +#[tokio::test] +async fn all_three_query_kinds_use_authoritative_default_and_explicit_views() { + let (dir, service, repository) = fixture().await; + let before = std::fs::read(dir.path().join("src/lib.rs")).unwrap(); + assert!(query(&service, &repository, None, search("sibling")) + .await + .unwrap() + .nodes + .is_empty()); + assert!(!query( + &service, + &repository, + Some(view("feature")), + search("sibling") + ) + .await + .unwrap() + .nodes + .is_empty()); + let base = query( + &service, + &repository, + None, + code("SiblingSecret|sibling_marker|disk_only"), + ) + .await + .unwrap(); + assert!(base.hits.is_empty()); + let base = base.content_result.unwrap(); + assert_eq!(base.total_matches, 0); + assert!(base.dir_facets.is_empty()); + let feature = query( + &service, + &repository, + Some(view("feature")), + code("SiblingSecret|sibling_marker"), + ) + .await + .unwrap(); + assert_eq!(feature.hits.len(), 1); + assert_eq!(feature.content_result.unwrap().total_matches, 2); + let base = query(&service, &repository, None, callers()).await.unwrap(); + assert_eq!(base.callers.len(), 1); + assert_eq!(base.callers[0].path, "src/lib.rs"); + assert_eq!(base.callers[0].name, "caller"); + assert_eq!(base.callers[0].line, 2); + let feature = query(&service, &repository, Some(view("feature")), callers()) + .await + .unwrap(); + assert_eq!(feature.callers.len(), 1); + assert_eq!(feature.callers[0].path, "src/hidden.rs"); + assert_eq!(feature.callers[0].name, "sibling_only"); + assert_eq!( + std::fs::read(dir.path().join("src/lib.rs")).unwrap(), + before + ); + let repo = Repository::open_readonly(dir.path()).unwrap(); + assert_eq!( + repo.desired_view_name(repo.require_working_copy_id().unwrap()) + .unwrap(), + "dev" + ); +} + +#[tokio::test] +async fn content_rpc_carries_filters_scoped_totals_facets_and_offsets() { + let (_dir, service, repository) = fixture().await; + let response = query( + &service, + &repository, + Some(view("dev")), + pb::query_graph_request::Query::ContentSearch(pb::ContentSearchQuery { + query: "TARGET|INHERITED_MARKER".into(), + paths: vec!["src/".into()], + path_filter: Some("**/*.rs".into()), + file_type: Some("rs".into()), + case_insensitive: true, + max_results: Some(1), + ..Default::default() + }), + ) + .await + .unwrap(); + let result = response.content_result.unwrap(); + assert_eq!(result.total_matches, 3); + assert_eq!(result.matches.len(), 1); + let m = &result.matches[0]; + assert_eq!(m.path, "src/lib.rs"); + assert_eq!(m.line_number, 1); + assert_eq!(m.byte_offset, 7); + assert_eq!((m.submatch_start, m.submatch_end), (7, 13)); + assert_eq!(result.dir_facets[0].count, 3); + assert_eq!(result.dir_facets[0].path, "src/lib.rs"); + + for (paths, exclude_type, max_results, expected_total) in [ + (vec!["src/lib".into()], None, Some(1), 0), // component boundary + (vec![], Some("rs".into()), Some(1), 0), + (vec![], None, Some(0), 3), + ] { + let response = query( + &service, + &repository, + None, + pb::query_graph_request::Query::ContentSearch(pb::ContentSearchQuery { + query: "target|inherited_marker".into(), + paths, + exclude_type, + max_results, + ..Default::default() + }), + ) + .await + .unwrap(); + let result = response.content_result.unwrap(); + assert!(result.matches.is_empty()); + assert_eq!(result.total_matches, expected_total); + } +} + +#[tokio::test] +async fn invalid_views_and_patterns_never_fall_back_to_ambient_results() { + let (_dir, service, repository) = fixture().await; + for kind in [search("target"), code("target"), callers()] { + assert!( + query(&service, &repository, Some(view("missing")), kind.clone()) + .await + .is_err() + ); + let mismatch = pb::ViewRef { + name: Some("dev".into()), + view_id: vec![1; 32], + }; + let error = query(&service, &repository, Some(mismatch), kind) + .await + .unwrap_err(); + assert_eq!(error.code(), tonic::Code::InvalidArgument); + } + assert!(query(&service, &repository, None, code("[")).await.is_err()); + // ID-only ViewRefs are accepted and resolve to the same explicit view. + let id = pb::ViewRef { + name: None, + view_id: blake3::hash(b"dev").as_bytes().to_vec(), + }; + assert!(!query(&service, &repository, Some(id), search("inherited")) + .await + .unwrap() + .nodes + .is_empty()); +} + +#[tokio::test] +async fn tonic_transport_serves_the_same_filtered_contract() { + use libatomic::daemon::{services_agent::KnowledgeImpl, state::DaemonState}; + use std::sync::Arc; + + let (dir, _local, _) = fixture().await; + let state = Arc::new(DaemonState::new()); + let repository = state + .register(dir.path().canonicalize().unwrap()) + .repository_ref(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(pb::knowledge_service_server::KnowledgeServiceServer::new( + KnowledgeImpl { state }, + )) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = stopped.await; + }, + ) + .await + .unwrap(); + }); + let mut client = + pb::knowledge_service_client::KnowledgeServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + for kind in [ + search("sibling"), + code("sibling_marker|SiblingSecret"), + callers(), + ] { + let base = client + .query_graph(pb::QueryGraphRequest { + repository: Some(repository.clone()), + query: Some(kind.clone()), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + let feature = client + .query_graph(pb::QueryGraphRequest { + repository: Some(repository.clone()), + query: Some(kind.clone()), + view: Some(view("feature")), + ..Default::default() + }) + .await + .unwrap() + .into_inner(); + match kind { + pb::query_graph_request::Query::KgSearch(_) => { + assert!(base.nodes.is_empty()); + assert!(!feature.nodes.is_empty()); + } + pb::query_graph_request::Query::ContentSearch(_) => { + assert_eq!(base.content_result.unwrap().total_matches, 0); + assert_eq!(feature.content_result.unwrap().total_matches, 2); + } + pb::query_graph_request::Query::Callers(_) => { + assert_eq!(base.callers[0].path, "src/lib.rs"); + assert_eq!(base.callers[0].name, "caller"); + assert_eq!(feature.callers[0].path, "src/hidden.rs"); + assert_eq!(feature.callers[0].name, "sibling_only"); + } + _ => unreachable!(), + } + } + drop(client); + stop.send(()).unwrap(); + server.await.unwrap(); +} diff --git a/tests/harness/14_vault_kg.sh b/tests/harness/14_vault_kg.sh index 667aff03..428d4cfd 100755 --- a/tests/harness/14_vault_kg.sh +++ b/tests/harness/14_vault_kg.sh @@ -11,6 +11,7 @@ # 7. Git import → KG enrichment → queries # 8. Memory and materialize # 9. Vault behavior in a repo initialized without --vault +# 10. View-filtered search, code and callers (including stale indexes) # # Every assertion checks both the exit code and the expected output shape. # A branch that would pass regardless of the command's outcome is a bug in @@ -241,9 +242,10 @@ create_file "src/main.rs" "fn main() { authenticate(); }" add_files "src/auth.rs" "src/main.rs" record_change "Add auth module" >/dev/null 2>&1 -# Run explicit KG enrichment — reports what was enriched +# The service reports the processed total; keep this strict about successful +# execution and a positive count rather than expecting the old local renderer. run_cmd atomic vault query enrich -assert_ran "KG enrich from VCS data" 'Enriched knowledge graph: [1-9][0-9]* item\(s\) processed\.' +assert_ran "KG enrich from VCS data" '^Enriched knowledge graph: [1-9][0-9]* item\(s\) processed\.$' # Search for workflow file resources (AST entities are not stored in the KG). run_cmd atomic vault query search "auth" --json @@ -321,7 +323,7 @@ else # Enrich KG from imported data run_cmd atomic vault query enrich - assert_ran "KG enrichment after git import" 'Enriched knowledge graph: [1-9][0-9]* item\(s\) processed\.' + assert_ran "KG enrichment after git import" '^Enriched knowledge graph: [1-9][0-9]* item\(s\) processed\.$' # Search should find content from the imported repo run_cmd atomic vault query search "uuid" --json @@ -385,6 +387,62 @@ assert_ran "vault list works after plain init" "memory/MEMORY\.md" run_cmd atomic vault goal list --json assert_ran "goal list is empty after plain init" '^\[\]$' +# ════════════════════════════════════════════════════════════════════════════ +# Section 10: View-Filtered Queries +# ════════════════════════════════════════════════════════════════════════════ + +begin_section "Queries: View Isolation" + +make_temp_repo "query-view-scope" +init_repo +create_file "src/lib.rs" $'pub fn target() {}\npub fn caller() { target(); }\n' +add_files "src/lib.rs" +assert_success "record base query source" atomic record -m "base query source" +assert_success "create inheriting query view" atomic view create query-feature --draft --parent dev +assert_success "switch to query feature" atomic view switch query-feature --force + +# Same paths/symbols, different signatures and call bodies: filtering paths alone +# would still expose the wrong view's source or caller relationships. +create_file "src/lib.rs" $'pub fn target(secret: SiblingSecret) {}\npub fn caller() {}\n' +create_file "src/sibling.rs" $'pub fn sibling_only() { target(); }\n// sibling_marker\n' +add_files "src/sibling.rs" +assert_success "record sibling query source" atomic record -m "sibling query source" +assert_success "enrich feature query data" atomic query enrich +assert_success "build working-copy index on feature" atomic query index --rebuild + +run_cmd atomic query code "SiblingSecret" --json +assert_ran "feature code includes its own signature" 'SiblingSecret' +run_cmd atomic query callers "src/lib.rs#target" --json +assert_ran "feature caller comes from sibling source" '"path": "src/sibling.rs"' +if [[ $RC -eq 0 ]] && ! echo "$OUT" | grep -qF '"name": "caller"'; then + _pass "feature does not retain base caller's removed call" +else + _fail "feature does not retain base caller's removed call" "exit $RC: $OUT" +fi + +assert_success "switch back to base query view" atomic view switch dev --force +create_file "src/lib.rs" $'pub fn disk_only() {}\n' + +run_cmd atomic query search "SiblingSecret" --json +assert_ran "base structural search excludes sibling signature" '^\[\]$' +run_cmd atomic query code 'SiblingSecret|sibling_marker|disk_only' --json +assert_ran "base code excludes sibling and unrecorded bytes" '"total_matches": 0' +run_cmd atomic query code 'TARGET' -i -g 'src/' -t rs -n 1 --json +assert_ran "code options preserve the recorded view's match count" '"total_matches": 2' +run_cmd atomic query callers "src/lib.rs#target" --json +assert_ran "base caller is recovered from recorded source" '"name": "caller"' +if [[ $RC -eq 0 ]] && ! echo "$OUT" | grep -qF '"path": "src/sibling.rs"'; then + _pass "base callers exclude sibling-only files" +else + _fail "base callers exclude sibling-only files" "exit $RC: $OUT" +fi + +# WORKING_COPIES, not this derived compatibility marker, selects the default. +printf '%s\n' 'query-feature' > .atomic/current_view +run_cmd atomic query code "SiblingSecret" --json +assert_ran "queries use registered view rather than compatibility marker" '"total_matches": 0' +assert_file_content "queries preserve unrecorded working-copy bytes" "src/lib.rs" 'pub fn disk_only() {}' + # ════════════════════════════════════════════════════════════════════════════ # Summary # ════════════════════════════════════════════════════════════════════════════ From ed8263367a8905fa70fdb7513de9485c29e09425 Mon Sep 17 00:00:00 2001 From: Bradley Hilton Date: Mon, 26 Oct 2026 22:10:54 +0100 Subject: [PATCH 2/5] test(query): follow the active imported view in KG checks --- tests/harness/14_vault_kg.sh | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/harness/14_vault_kg.sh b/tests/harness/14_vault_kg.sh index 428d4cfd..f66007a6 100755 --- a/tests/harness/14_vault_kg.sh +++ b/tests/harness/14_vault_kg.sh @@ -54,12 +54,27 @@ assert_ran() { fi } -# Discover a canonical view IRI from the query result, then require a real -# incoming membership edge. An empty neighbors response is not a passing graph. +# Discover the registered current view and its canonical IRI, then require a real +# incoming membership edge. Import may select a different view than init created. +# An empty neighbors response is not a passing graph. assert_view_membership() { local desc="$1" - local view_id - run_cmd atomic vault query search main --kind 'https://atomic.dev/ns/workflow#View' --json + local view_name view_id + run_cmd atomic view list --json + if [[ $RC -ne 0 ]]; then + _fail "$desc" "view list exited $RC: $OUT" + return + fi + if ! view_name="$(echo "$OUT" | python3 -c ' +import json, sys +name = json.load(sys.stdin)["current_view"] +assert isinstance(name, str) and name +print(name) +')"; then + _fail "$desc" "registered current view missing: $OUT" + return + fi + run_cmd atomic vault query search "$view_name" --kind 'https://atomic.dev/ns/workflow#View' --json if [[ $RC -ne 0 ]]; then _fail "$desc" "view search exited $RC: $OUT" return @@ -67,9 +82,9 @@ assert_view_membership() { if ! view_id="$(echo "$OUT" | python3 -c ' import json, sys nodes = json.load(sys.stdin) -print(next(node["id"] for node in nodes if node["label"] == "main" and "https://atomic.dev/ns/workflow#View" in node["types"])) -')"; then - _fail "$desc" "canonical main view missing: $OUT" +print(next(node["id"] for node in nodes if node["label"] == sys.argv[1] and "https://atomic.dev/ns/workflow#View" in node["types"])) +' "$view_name")"; then + _fail "$desc" "canonical $view_name view missing: $OUT" return fi run_cmd atomic vault query neighbors "$view_id" --json From 8cf6fbb165e4396e7127e0ba445c461b6b5dc57a Mon Sep 17 00:00:00 2001 From: Bradley Hilton Date: Mon, 26 Oct 2026 22:18:02 +0100 Subject: [PATCH 3/5] test(query): box service helper errors for Clippy --- libatomic/tests/query_view_service_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libatomic/tests/query_view_service_test.rs b/libatomic/tests/query_view_service_test.rs index 5909310b..5d91b726 100644 --- a/libatomic/tests/query_view_service_test.rs +++ b/libatomic/tests/query_view_service_test.rs @@ -84,7 +84,7 @@ async fn query( repository: &pb::RepositoryRef, view: Option, query: pb::query_graph_request::Query, -) -> Result { +) -> Result> { let bytes = service .call( "/atomic.KnowledgeService/QueryGraph", From 3b3f1c5e6fd13f1fbd6d34d57aa274d6fc818db4 Mon Sep 17 00:00:00 2001 From: Bradley Hilton Date: Tue, 27 Oct 2026 08:34:39 +0100 Subject: [PATCH 4/5] fix: apply KG class filters before candidate pool truncation --- README.md | 11 + atomic-cli/src/commands/query/mod.rs | 128 +++++--- atomic-cli/src/commands/rpc.rs | 5 +- atomic-cli/tests/query_view_filtering_test.rs | 125 ++++++++ .../src/repository/tests/query_view_tests.rs | 214 +++++++++++++ .../src/repository/vault_triples.rs | 47 ++- .../knowledge/knowledge_messages.proto | 13 +- libatomic/src/daemon/services_agent.rs | 37 +-- libatomic/tests/query_view_service_test.rs | 280 +++++++++++++++++- 9 files changed, 785 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index e1451681..3fad0a21 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,17 @@ select a view explicitly; omitting it uses the working copy's registered desired view. Other query verbs retain their existing scope semantics. The separate working-copy content index is still available to its existing consumers. +For `query search`, `--kind` matches an exact, case-sensitive absolute class IRI +against **any asserted type**, before candidate-pool truncation. An omitted or +empty kind means no filter; an unknown class matches nothing. `--pool` defaults +to 5000 and is a retention budget, normalized to at least the result limit—not a +cap on which eligible candidates are examined. Without a kind filter, a larger +pool enables cross-type diversity. With a kind filter, results are the highest- +ranked matching nodes without cross-type diversity, so even `--pool 1` cannot hide +their top result. CLI/service search defaults to 10 results; a service request +limit of zero (including CLI `--limit 0`) uses that default. Direct repository +search APIs instead interpret a zero limit literally and return no nodes. + **SHACL validation** runs locally in the CLI using PurRDF. Builtin profiles and enterprise-authored `.atomic/shapes/*.ttl` files use the same engine: diff --git a/atomic-cli/src/commands/query/mod.rs b/atomic-cli/src/commands/query/mod.rs index 01360ee8..48d70acb 100644 --- a/atomic-cli/src/commands/query/mod.rs +++ b/atomic-cli/src/commands/query/mod.rs @@ -1159,23 +1159,21 @@ pub struct QueryNodes { /// Search query text (keyword search over node IDs, labels, and summaries). pub query: String, - /// Maximum results. + /// Maximum results (0 uses the service default of 10). #[arg(long, short = 'k', default_value = "10")] pub limit: usize, /// Filter by canonical class IRI (for example, ). /// - /// When set, only nodes asserting this type are returned. - /// Without this flag, all types are returned. + /// Matches any asserted type exactly and case-sensitively, before pool limits. + /// Omitted or empty: all types. An unknown class returns no results. #[arg(long, short = 't')] pub kind: Option, - /// Candidate pool size. + /// Candidate retention budget (default: 5000; at least --limit). /// - /// How many candidates to score before selecting the top results. - /// A larger pool surfaces lower-scored node kinds (e.g. changes) - /// that would otherwise be crowded out by files with content matches. - /// Default: same as --limit. + /// Without --kind, a larger pool allows diversity across types. With --kind, + /// only matching nodes compete; a small pool cannot hide their top results. #[arg(long, short = 'p')] pub pool: Option, @@ -1199,37 +1197,7 @@ impl Command for QueryNodes { .desired_view_name(working_copy) .map_err(CliError::Repository)?; - // When filtering by kind, we need enough candidates of that kind - // to survive the diversity selection. Ask for a large pool so the - // heap retains many candidates, then post-filter to the target kind. - let (fetch_limit, pool) = if self.kind.is_some() { - let pool_val = self.pool.unwrap_or(5000); - // fetch_limit = pool so all pool candidates become results - (pool_val, Some(pool_val)) - } else { - // Without a kind filter, request a generous candidate pool so the - // diversity-aware selection in `vault_kg_search` engages. Otherwise - // (pool == limit) it falls back to plain top-N by score, and once a - // content index exists, high-scoring `file`/`module` nodes crowd - // out lower-scored vault nodes (intents, todos, sessions, - // memories), so an enriched repo would only ever return files. - (self.limit, Some(self.pool.unwrap_or(5000))) - }; - - let all_nodes = repo - .vault_kg_search_on_view(&view, &self.query, fetch_limit, pool) - .map_err(CliError::Repository)?; - - let nodes: Vec<_> = if let Some(ref kind_filter) = self.kind { - let k = kind_filter.to_owned(); - all_nodes - .into_iter() - .filter(|n| n.types.contains(&k)) - .take(self.limit) - .collect() - } else { - all_nodes.into_iter().take(self.limit).collect() - }; + let nodes = self.search_on_view(&repo, &view)?; if self.json { println!("{}", serde_json::to_string_pretty(&nodes).unwrap()); @@ -1257,6 +1225,88 @@ impl Command for QueryNodes { } } +impl QueryNodes { + fn search_on_view(&self, repo: &Repository, view: &str) -> CliResult> { + repo.vault_kg_search_on_view_with_kind( + view, + &self.query, + self.limit, + Some(self.pool.unwrap_or(5000)), + self.kind.as_deref(), + ) + .map_err(CliError::Repository) + } +} + +#[cfg(test)] +mod search_tests { + use super::*; + use atomic_core::kg::iri; + use atomic_core::pristine::VaultEntryType; + + #[test] + fn direct_search_helper_filters_kind_before_pool_limits() { + let dir = tempfile::tempdir().unwrap(); + let repo = Repository::init(dir.path()).unwrap(); + let memory = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + repo.vault_store( + &format!("memory/{memory}.md"), + VaultEntryType::Memory, + b":::memory\nFallback regression.\n:::\n".to_vec(), + serde_json::json!({"id": memory, "kind": "context"}).to_string(), + ) + .unwrap(); + std::fs::write(dir.path().join("pool.rs"), format!("// {memory}\n")).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + repo.add(wc, "pool.rs", Default::default()).unwrap(); + repo.record( + wc, + atomic_core::change::ChangeHeader::new("record fallback fixture"), + atomic_repository::RecordOptions::new() + .with_all(true) + .save_to_store(true) + .apply_after_record(true), + ) + .unwrap(); + let memory_id = iri::mint_memory(&repo.project_id().unwrap(), memory); + for pool in [0, 1, 100] { + let args = QueryNodes::try_parse_from([ + "search", + memory, + "--kind", + Class::Memory.iri(), + "--pool", + &pool.to_string(), + "--limit", + "1", + ]) + .unwrap(); + let nodes = args.search_on_view(&repo, "dev").unwrap(); + assert_eq!(nodes.len(), 1, "pool={pool}"); + assert_eq!(nodes[0].id, memory_id); + let args = QueryNodes { limit: 0, ..args }; + assert!(args.search_on_view(&repo, "dev").unwrap().is_empty()); + } + for kind in [None, Some(String::new()), Some("urn:unknown:class".into())] { + let args = QueryNodes { + query: memory.into(), + limit: 1, + pool: Some(1), + kind: kind.clone(), + json: true, + }; + let nodes = args.search_on_view(&repo, "dev").unwrap(); + if kind.as_deref() == Some("urn:unknown:class") { + assert!(nodes.is_empty()); + } else { + assert_eq!(nodes.len(), 1); + assert!(nodes[0].has_type(Class::File)); + } + assert!(args.search_on_view(&repo, "missing").is_err()); + } + } +} + /// Get the neighborhood of a KG node. #[derive(Parser, Debug)] pub struct QueryNeighbors { diff --git a/atomic-cli/src/commands/rpc.rs b/atomic-cli/src/commands/rpc.rs index 859cb5dd..9c7f5661 100644 --- a/atomic-cli/src/commands/rpc.rs +++ b/atomic-cli/src/commands/rpc.rs @@ -1192,9 +1192,8 @@ pub fn memory_list(args: &super::memory::list::MemoryList) -> CliResult { /// `atomic vault query search` over QueryGraph (KgSearch) — every form: /// the wire's KGNode now carries the full domain fields (source, /// metadata), so the JSON serializes the same shape the local body -/// emits; `--kind` rides the request (the handler filters the candidate -/// pool server-side, taking the limit after the filter — the local -/// post-filter order) and `--pool` names the candidate pool. The plain +/// emits; `--kind` rides the request (the repository filters projected nodes +/// before ranking/pool limits) and `--pool` names the retention budget. The plain /// render reads the same kind/id/summary columns. pub fn query_search(args: &super::query::QueryNodes) -> CliResult { let Some(session) = Service::open()? else { diff --git a/atomic-cli/tests/query_view_filtering_test.rs b/atomic-cli/tests/query_view_filtering_test.rs index a1da4df0..0c0dbac0 100644 --- a/atomic-cli/tests/query_view_filtering_test.rs +++ b/atomic-cli/tests/query_view_filtering_test.rs @@ -141,3 +141,128 @@ fn query_verbs_render_filtered_json_and_human_output_over_the_service() { "pub fn disk_only() {}\n" ); } + +#[test] +fn search_kind_with_tiny_pool_returns_the_matching_memory_over_the_service() { + let dir = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + drop(Repository::init(dir.path()).unwrap()); + let created = run( + dir.path(), + home.path(), + &[ + "memory", + "new", + "--kind", + "context", + "--text", + "Pool starvation regression.", + ], + ); + assert!( + created.status.success(), + "{}", + String::from_utf8_lossy(&created.stderr) + ); + let created = String::from_utf8(created.stdout).unwrap(); + let memory = created + .lines() + .find_map(|line| line.strip_prefix("Created memory: ")) + .unwrap(); + let repo = Repository::open(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + let memory_id = atomic_core::kg::iri::mint_memory(&repo.project_id().unwrap(), memory); + for index in 0..12 { + let path = format!("pool_{index}.rs"); + std::fs::write(dir.path().join(&path), format!("// {memory}\n")).unwrap(); + repo.add(wc, &path, TrackingOptions::default()).unwrap(); + } + record(&repo, "higher scored file candidates"); + drop(repo); + let class = atomic_core::kg::ontology::Class::Memory.iri(); + let mut expected = None; + for pool in ["0", "1", "100"] { + let nodes = json( + dir.path(), + home.path(), + &[ + "query", "search", memory, "--kind", class, "--pool", pool, "--limit", "1", + "--json", + ], + ); + assert_eq!(nodes.as_array().unwrap().len(), 1, "pool={pool}"); + assert_eq!(nodes[0]["id"], memory_id); + assert!(nodes[0]["types"] + .as_array() + .unwrap() + .iter() + .any(|ty| ty == class)); + if let Some(expected) = &expected { + assert_eq!(&nodes, expected, "pool changed filtered result"); + } else { + expected = Some(nodes); + } + } + let nodes = json( + dir.path(), + home.path(), + &[ + "query", "search", memory, "--kind", class, "--pool", "0", "--limit", "0", "--json", + ], + ); + assert_eq!( + nodes, + expected.unwrap(), + "CLI zero uses the service default, not literal zero" + ); + let plain = json( + dir.path(), + home.path(), + &[ + "query", "search", memory, "--pool", "1", "--limit", "1", "--json", + ], + ); + assert_eq!( + plain[0]["types"][0], + atomic_core::kg::ontology::Class::File.iri() + ); + let empty_kind = json( + dir.path(), + home.path(), + &[ + "query", "search", memory, "--kind", "", "--pool", "1", "--limit", "1", "--json", + ], + ); + assert_eq!(empty_kind, plain); + for kind in [ + "Memory", + "https://atomic.dev/ns/workflow#memory", + "urn:unknown:class", + ] { + assert_eq!( + json( + dir.path(), + home.path(), + &["query", "search", memory, "--kind", kind, "--pool", "1", "--json",] + ), + serde_json::json!([]) + ); + } + let output = run( + dir.path(), + home.path(), + &[ + "query", "search", memory, "--kind", class, "--pool", "1", "--limit", "1", + ], + ); + assert!(output.status.success()); + let output = String::from_utf8(output.stdout).unwrap(); + assert!( + output.contains(&memory_id) && output.contains("1 result(s)."), + "{output}" + ); + assert!(!dir + .path() + .join(".atomic/content-index/manifest.json") + .exists()); +} diff --git a/atomic-repository/src/repository/tests/query_view_tests.rs b/atomic-repository/src/repository/tests/query_view_tests.rs index 49f7530d..6f401349 100644 --- a/atomic-repository/src/repository/tests/query_view_tests.rs +++ b/atomic-repository/src/repository/tests/query_view_tests.rs @@ -279,8 +279,222 @@ fn invalid_dependency_closure_fails_closed_for_every_query() { .unwrap_err(), repo.query_callers_on_view("feature", "src/lib.rs#target") .unwrap_err(), + repo.vault_kg_search_on_view_with_kind( + "feature", + "sibling", + 0, + Some(0), + Some("urn:unknown:class"), + ) + .unwrap_err(), ]; for error in errors { assert!(error.to_string().contains(&missing.to_base32()), "{error}"); } } + +#[test] +fn query_kind_filter_is_pool_independent_and_matches_every_asserted_type() { + let (temp, repo) = create_temp_repo(); + let marker = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + repo.vault_store( + &format!("memory/{marker}.md"), + atomic_core::pristine::VaultEntryType::Memory, + b":::memory\nPool regression.\n:::\n".to_vec(), + serde_json::json!({"id": marker, "kind": "context"}).to_string(), + ) + .unwrap(); + std::fs::create_dir_all(temp.path().join("src")).unwrap(); + for index in 0..8 { + // Both filename and content match, so FTS/projected/content candidates + // must be deduplicated and must not displace an eligible Memory. + let path = format!("src/{marker}_{index}.rs"); + std::fs::write(temp.path().join(&path), format!("// {marker}\n")).unwrap(); + repo.add(&path, TrackingOptions::default()).unwrap(); + } + record(&repo, "record pool files"); + let project = repo.project_id().unwrap(); + let memory_id = iri::mint_memory(&project, marker); + let mut txn = repo.pristine().write_txn().unwrap(); + for index in 0..4 { + txn.upsert_kg_node( + &KgNode::new( + iri::mint_goal(&project, &format!("pool-{index}")), + Class::Goal, + marker, + "vault", + ) + .with_type(Class::Memory), + ) + .unwrap(); + } + txn.commit().unwrap(); + let exhaustive = repo + .vault_kg_search_on_view("dev", marker, usize::MAX, None) + .unwrap(); + let expected_memories: Vec<_> = exhaustive + .iter() + .filter(|node| node.has_type(Class::Memory)) + .cloned() + .collect(); + assert_eq!(expected_memories.len(), 5); + assert!(expected_memories.iter().any(|node| node.id == memory_id)); + assert!(expected_memories + .iter() + .any(|node| node.primary_type() != Class::Memory.iri())); + + for limit in [0, 1, 3, 20] { + for pool in [None, Some(0), Some(1), Some(3), Some(100)] { + for kind in [ + None, + Some(""), + Some(Class::Memory.iri()), + Some(Class::File.iri()), + Some("Memory"), + Some("https://atomic.dev/ns/workflow#memory"), + Some("urn:unknown:class"), + ] { + let actual = repo + .vault_kg_search_on_view_with_kind("dev", marker, limit, pool, kind) + .unwrap(); + let expected: Vec<_> = if let Some(kind) = kind.filter(|kind| !kind.is_empty()) { + exhaustive + .iter() + .filter(|node| node.types.iter().any(|ty| ty == kind)) + .take(limit) + .cloned() + .collect() + } else { + repo.vault_kg_search_on_view("dev", marker, limit, pool) + .unwrap() + }; + assert_eq!( + actual, expected, + "limit={limit}, pool={pool:?}, kind={kind:?}" + ); + let ids: std::collections::HashSet<_> = + actual.iter().map(|node| &node.id).collect(); + assert_eq!(ids.len(), actual.len(), "duplicate results: {actual:?}"); + } + } + } + for (view, query) in [("dev", "no_matching_token"), ("missing", marker)] { + for limit in [0, 3] { + for kind in [Class::Memory.iri(), "urn:unknown:class"] { + let result = + repo.vault_kg_search_on_view_with_kind(view, query, limit, Some(0), Some(kind)); + if view == "missing" { + assert!(result.is_err(), "unknown view must not short-circuit"); + } else { + assert!(result.unwrap().is_empty()); + } + } + } + } +} + +#[test] +fn query_kind_filter_preserves_view_projection_before_type_matching() { + let (temp, mut repo, feature_hash) = fixture(); + repo.create_draft_view("child-filtered", "feature").unwrap(); + repo.create_view_with_identity("tip-filtered", ViewScope::Shared, None) + .unwrap(); + let mut txn = repo.pristine().write_txn().unwrap(); + let mut view = txn.get_view("tip-filtered").unwrap().unwrap(); + let id = txn.get_internal(&feature_hash).unwrap().unwrap(); + txn.put_change(&mut view, id, &feature_hash).unwrap(); + txn.update_view(&view).unwrap(); + txn.commit().unwrap(); + let before = std::fs::read(temp.path().join("src/lib.rs")).unwrap(); + for view in ["dev", "feature", "child-filtered", "tip-filtered"] { + for query in [ + "sibling", + "SiblingSecret", + "HiddenSiblingAuthor", + "inherited", + ] { + let exhaustive = repo + .vault_kg_search_on_view(view, query, 100, None) + .unwrap(); + for class in [Class::File, Class::Change, Class::Identity, Class::View] { + let expected: Vec<_> = exhaustive + .iter() + .filter(|node| node.has_type(class)) + .take(1) + .cloned() + .collect(); + let actual = repo + .vault_kg_search_on_view_with_kind(view, query, 1, Some(1), Some(class.iri())) + .unwrap(); + assert_eq!( + actual, expected, + "view={view}, query={query}, class={class:?}" + ); + } + } + } + assert_eq!( + std::fs::read(temp.path().join("src/lib.rs")).unwrap(), + before + ); + assert_eq!( + repo.desired_view_name(repo.working_copy()).unwrap(), + "feature" + ); +} + +#[test] +fn query_no_kind_ranking_and_diversity_retain_their_golden_order() { + let (_temp, repo) = create_temp_repo(); + repo.init_kg().unwrap(); + let project = repo.project_id().unwrap(); + let mut goals = Vec::new(); + let mut memories = Vec::new(); + let mut txn = repo.pristine().write_txn().unwrap(); + for index in 0..4 { + let id = iri::mint_goal(&project, &format!("golden-{index}")); + txn.upsert_kg_node(&KgNode::new(&id, Class::Goal, "goldenmarker", "vault")) + .unwrap(); + goals.push(id); + } + for index in 0..2 { + let id = iri::mint_memory(&project, &format!("golden-{index}")); + txn.upsert_kg_node(&KgNode::new(&id, Class::Memory, "goldenmarker", "vault")) + .unwrap(); + memories.push(id); + } + txn.commit().unwrap(); + for (pool, expected) in [ + (None, goals.clone()), + (Some(0), goals.clone()), + (Some(4), goals.clone()), + (Some(100), [goals[..2].to_vec(), memories].concat()), + ] { + let plain = repo + .vault_kg_search_on_view("dev", "goldenmarker", 4, pool) + .unwrap(); + assert_eq!( + plain.iter().map(|node| node.id.clone()).collect::>(), + expected + ); + assert_eq!( + repo.vault_kg_search("goldenmarker", 4, pool).unwrap(), + plain + ); + for kind in [None, Some("")] { + assert_eq!( + repo.vault_kg_search_on_view_with_kind("dev", "goldenmarker", 4, pool, kind) + .unwrap(), + plain, + ); + } + } + assert_eq!( + repo.vault_kg_search_by_kind("goldenmarker", 4, Class::Goal) + .unwrap() + .iter() + .map(|node| node.id.clone()) + .collect::>(), + goals, + ); +} diff --git a/atomic-repository/src/repository/vault_triples.rs b/atomic-repository/src/repository/vault_triples.rs index 3f3a267f..787d9ed2 100644 --- a/atomic-repository/src/repository/vault_triples.rs +++ b/atomic-repository/src/repository/vault_triples.rs @@ -240,7 +240,7 @@ impl Repository { limit: usize, pool: Option, ) -> Result, RepositoryError> { - self.vault_kg_search_inner(query, limit, pool, None) + self.vault_kg_search_inner(query, limit, pool, None, None) } /// Search VCS nodes and source content through the requested view's @@ -252,9 +252,43 @@ impl Repository { query: &str, limit: usize, pool: Option, + ) -> Result, RepositoryError> { + self.vault_kg_search_on_view_with_kind(view_name, query, limit, pool, None) + } + + /// Search a recorded view, optionally filtering by an exact asserted class IRI. + /// + /// View projection and class filtering precede ranking and pool truncation. + /// The class is matched against every asserted type, case-sensitively; an + /// absent or empty class leaves the search unfiltered. Unknown classes match + /// nothing. Filtered results are the top `limit` nodes by score, without + /// cross-type diversity, regardless of `pool` (retained at least to `limit`). + /// A zero `limit` returns no nodes but still validates the requested view. + /// Vault-only nodes remain shared; no index is built or refreshed. + /// + /// # Examples + /// + /// ```no_run + /// use atomic_core::kg::ontology::Class; + /// use atomic_repository::{Repository, RepositoryError}; + /// # fn main() -> Result<(), RepositoryError> { + /// let repo = Repository::open_readonly("path/to/repository")?; + /// let memories = repo.vault_kg_search_on_view_with_kind( + /// "dev", "leases", 10, Some(1), Some(Class::Memory.iri()), + /// )?; + /// # Ok(()) + /// # } + /// ``` + pub fn vault_kg_search_on_view_with_kind( + &self, + view_name: &str, + query: &str, + limit: usize, + pool: Option, + kind: Option<&str>, ) -> Result, RepositoryError> { let scope = self.query_view(view_name)?; - self.vault_kg_search_inner(query, limit, pool, Some(&scope)) + self.vault_kg_search_inner(query, limit, pool, Some(&scope), kind) } fn vault_kg_search_inner( @@ -263,7 +297,9 @@ impl Repository { limit: usize, pool: Option, scope: Option<&super::query::QueryView>, + kind: Option<&str>, ) -> Result, RepositoryError> { + let kind = kind.filter(|kind| !kind.is_empty()); let txn = self.pristine.read_txn()?; let mut rows = BTreeMap::new(); let mut content = HashMap::::new(); @@ -331,6 +367,11 @@ impl Repository { } let mut ranked = Vec::new(); for (_, (mut node, hits)) in rows { + // All candidate sources, including content-only File stubs, obey + // the class predicate before competing for retained result slots. + if kind.is_some_and(|kind| !node.types.iter().any(|ty| ty == kind)) { + continue; + } let count = node .path() .and_then(|path| content.get(path)) @@ -353,7 +394,7 @@ impl Repository { } ranked.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| a.1.id.cmp(&b.1.id))); ranked.truncate(pool.unwrap_or(limit).max(limit)); - if pool.is_some_and(|pool| pool > limit) { + if kind.is_none() && pool.is_some_and(|pool| pool > limit) { let mut per_type = HashMap::::new(); let mut selected = Vec::new(); let mut remaining = Vec::new(); diff --git a/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto b/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto index 780c9965..6bd5b9fa 100644 --- a/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto +++ b/libatomic/proto/atomic/services/knowledge/knowledge_messages.proto @@ -44,7 +44,7 @@ message GetWorkflowSnapshotResponse { message QueryGraphRequest { RepositoryRef repository = 1; - uint32 limit = 2; // default 10-20 per kind today + uint32 limit = 2; // kg_search: 0 uses 10; other kinds use their defaults oneof query { ContentSearchQuery content_search = 3; // `query code` (recorded view bytes) KgSearchQuery kg_search = 4; // `query search` @@ -80,10 +80,15 @@ message ContentSearchQuery { message KgSearchQuery { string query = 1; // `query search --kind` (add-only): keep only nodes asserting this absolute - // class IRI, applied over the candidate pool before the limit. + // class IRI. Exact, case-sensitive membership in any asserted type is applied + // AFTER view projection and BEFORE pool truncation or the result limit. + // Absent/empty means no filter; an unknown class matches nothing. optional string kind = 2; - // The candidate pool size (add-only): how many candidates to score - // before selecting the top results (the CLI's --pool, default 5000). + // Candidate retention budget (the CLI's --pool, default 5000), normalized + // to at least the effective result limit. All eligible candidates are scored. + // Without kind, a larger pool supports cross-type diversity. With kind, + // return the highest-ranked matching nodes without cross-type diversity; + // a small pool (including zero) cannot starve matching results. optional uint32 pool = 3; } message EntityQuery { diff --git a/libatomic/src/daemon/services_agent.rs b/libatomic/src/daemon/services_agent.rs index 1df33054..6799e58c 100644 --- a/libatomic/src/daemon/services_agent.rs +++ b/libatomic/src/daemon/services_agent.rs @@ -3555,31 +3555,18 @@ impl knowledge_service_server::KnowledgeService for KnowledgeImpl { QueryOneof::KgSearch(search) => { let repo = handle.repository_readonly()?; let view = query_view_name(&repo, requested_view.as_ref())?; - // The candidate pool the CLI's --pool names (default 5000) - // — the pool is what lets lower-scored kinds survive the - // diversity selection, so it rides the request. - let pool = search.pool.map(|pool| pool as usize).unwrap_or(5000); - let nodes = if let Some(kind_filter) = - search.kind.clone().filter(|kind| !kind.is_empty()) - { - // The --kind form: search the full pool, keep the - // kind's nodes, take the limit AFTER filtering — the - // local post-filter order. - let kind = kind_filter; - repo.vault_kg_search_on_view(&view, &search.query, pool, Some(pool)) - .map_err(repository_error)? - .into_iter() - .filter(|node| node.types.contains(&kind)) - .take(limit) - .map(|node| crate::daemon::services_query::kg_node_proto(node.clone())) - .collect::>() - } else { - repo.vault_kg_search_on_view(&view, &search.query, limit, Some(pool)) - .map_err(repository_error)? - .into_iter() - .map(|node| crate::daemon::services_query::kg_node_proto(node.clone())) - .collect() - }; + let nodes = repo + .vault_kg_search_on_view_with_kind( + &view, + &search.query, + limit, + Some(search.pool.map(|pool| pool as usize).unwrap_or(5000)), + search.kind.as_deref(), + ) + .map_err(repository_error)? + .into_iter() + .map(crate::daemon::services_query::kg_node_proto) + .collect(); Ok::<_, Status>(QueryGraphResponse { api_version: 1, callers: Vec::new(), diff --git a/libatomic/tests/query_view_service_test.rs b/libatomic/tests/query_view_service_test.rs index 5d91b726..6483fbde 100644 --- a/libatomic/tests/query_view_service_test.rs +++ b/libatomic/tests/query_view_service_test.rs @@ -2,6 +2,8 @@ //! LocalService and the daemon, including explicit non-materialized views. use atomic_core::change::ChangeHeader; +use atomic_core::kg::{iri, ontology::Class}; +use atomic_core::pristine::{KgMutTxnT, KgTxnT, MutTxnT, VaultEntryType}; use atomic_repository::{RecordOptions, Repository, TrackingOptions}; use libatomic::{proto as pb, LocalService}; use prost::Message; @@ -84,6 +86,16 @@ async fn query( repository: &pb::RepositoryRef, view: Option, query: pb::query_graph_request::Query, +) -> Result> { + query_with_limit(service, repository, view, query, 1).await +} + +async fn query_with_limit( + service: &LocalService, + repository: &pb::RepositoryRef, + view: Option, + query: pb::query_graph_request::Query, + limit: u32, ) -> Result> { let bytes = service .call( @@ -92,7 +104,7 @@ async fn query( repository: Some(repository.clone()), view, query: Some(query), - limit: 1, + limit, } .encode_to_vec(), ) @@ -348,3 +360,269 @@ async fn tonic_transport_serves_the_same_filtered_contract() { stop.send(()).unwrap(); server.await.unwrap(); } + +async fn pool_fixture() -> (tempfile::TempDir, LocalService, pb::RepositoryRef, String) { + let (dir, service, repository) = fixture().await; + let repo = Repository::open(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + let memory = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + repo.vault_store( + &format!("memory/{memory}.md"), + VaultEntryType::Memory, + b":::memory\nPool starvation regression.\n:::\n".to_vec(), + serde_json::json!({"id": memory, "kind": "context", "status": "active"}).to_string(), + ) + .unwrap(); + let memory_id = iri::mint_memory(&repo.project_id().unwrap(), memory); + for index in 0..12 { + let path = format!("src/pool_{index}.rs"); + std::fs::write(dir.path().join(&path), format!("// {memory}\n")).unwrap(); + repo.add(wc, &path, TrackingOptions::default()).unwrap(); + } + record(&repo, "record higher-scored pool candidates"); + drop(repo); + (dir, service, repository, memory_id) +} + +#[tokio::test] +async fn kind_filter_precedes_the_mixed_candidate_pool() { + let (_dir, service, repository, memory_id) = pool_fixture().await; + for pool in [100, 1, 0] { + let response = query( + &service, + &repository, + None, + pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: "01ARZ3NDEKTSV4RRFFQ69G5FAV".into(), + kind: Some(Class::Memory.iri().into()), + pool: Some(pool), + }), + ) + .await + .unwrap(); + assert_eq!(response.nodes.len(), 1, "pool={pool}: {response:?}"); + assert_eq!(response.nodes[0].id, memory_id, "pool={pool}"); + assert!(response.nodes[0] + .types + .iter() + .any(|ty| ty == Class::Memory.iri())); + } +} + +#[tokio::test] +async fn kind_rpc_preserves_nonprimary_types_zero_defaults_and_view_errors() { + let (dir, service, repository, memory_id) = pool_fixture().await; + let repo = Repository::open(dir.path()).unwrap(); + let project = repo.project_id().unwrap(); + let mut txn = repo.pristine().write_txn().unwrap(); + let memory = txn + .get_kg_node(&memory_id) + .unwrap() + .unwrap() + .with_type(Class::Goal); + assert_ne!(memory.primary_type(), Class::Memory.iri()); + txn.upsert_kg_node(&memory).unwrap(); + let mut default_ids = Vec::new(); + for index in 0..12 { + let id = iri::mint_memory(&project, &format!("default-{index:02}")); + txn.upsert_kg_node(&atomic_core::KgNode::new( + &id, + Class::Memory, + "defaultmarker", + "vault", + )) + .unwrap(); + default_ids.push(id); + } + txn.commit().unwrap(); + let before = std::fs::read(dir.path().join("src/lib.rs")).unwrap(); + let graph_before = serde_json::to_value(repo.workflow_snapshot().unwrap()).unwrap(); + let index_before = std::fs::read(dir.path().join(".atomic/content-index/manifest.json")).ok(); + drop(repo); + for requested_view in [None, Some(view("dev")), Some(view("feature"))] { + for pool in [None, Some(0), Some(1), Some(100)] { + let result = query( + &service, + &repository, + requested_view.clone(), + pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: "01ARZ3NDEKTSV4RRFFQ69G5FAV".into(), + kind: Some(Class::Memory.iri().into()), + pool, + }), + ) + .await + .unwrap(); + assert_eq!(result.nodes.len(), 1, "pool={pool:?}"); + assert_eq!(result.nodes[0].id, memory_id); + assert_eq!(result.nodes[0].types, memory.types); + } + } + for kind in [ + None, + Some(""), + Some(Class::Memory.iri()), + Some("urn:unknown:class"), + ] { + let result = query_with_limit( + &service, + &repository, + None, + pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: "defaultmarker".into(), + kind: kind.map(str::to_owned), + pool: Some(0), + }), + 0, + ) + .await + .unwrap(); + let expected = if kind == Some("urn:unknown:class") { + vec![] + } else { + default_ids[..10].to_vec() + }; + assert_eq!( + result + .nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(), + expected + ); + } + for term in ["sibling", "SiblingSecret"] { + let kind = pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: term.into(), + kind: Some(Class::File.iri().into()), + pool: Some(1), + }); + assert!(query(&service, &repository, None, kind.clone()) + .await + .unwrap() + .nodes + .is_empty()); + assert!( + !query(&service, &repository, Some(view("feature")), kind.clone()) + .await + .unwrap() + .nodes + .is_empty() + ); + assert!( + query(&service, &repository, Some(view("missing")), kind.clone()) + .await + .is_err() + ); + assert_eq!( + query( + &service, + &repository, + Some(pb::ViewRef { + name: Some("dev".into()), + view_id: vec![1; 32] + }), + kind + ) + .await + .unwrap_err() + .code(), + tonic::Code::InvalidArgument + ); + } + assert_eq!( + std::fs::read(dir.path().join("src/lib.rs")).unwrap(), + before + ); + assert_eq!( + std::fs::read(dir.path().join(".atomic/content-index/manifest.json")).ok(), + index_before + ); + let repo = Repository::open_readonly(dir.path()).unwrap(); + assert_eq!( + serde_json::to_value(repo.workflow_snapshot().unwrap()).unwrap(), + graph_before + ); + assert_eq!( + repo.desired_view_name(repo.require_working_copy_id().unwrap()) + .unwrap(), + "dev" + ); +} + +#[tokio::test] +async fn tonic_kind_filter_is_applied_before_pool_and_result_limits() { + use libatomic::daemon::{services_agent::KnowledgeImpl, state::DaemonState}; + use std::sync::Arc; + + let (dir, local, local_repository, memory_id) = pool_fixture().await; + let repo = Repository::open(dir.path()).unwrap(); + let mut txn = repo.pristine().write_txn().unwrap(); + let memory = txn + .get_kg_node(&memory_id) + .unwrap() + .unwrap() + .with_type(Class::Goal); + txn.upsert_kg_node(&memory).unwrap(); + txn.commit().unwrap(); + drop(repo); + let state = Arc::new(DaemonState::new()); + let repository = state + .register(dir.path().canonicalize().unwrap()) + .repository_ref(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(pb::knowledge_service_server::KnowledgeServiceServer::new( + KnowledgeImpl { state }, + )) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = stopped.await; + }, + ) + .await + .unwrap(); + }); + let mut client = + pb::knowledge_service_client::KnowledgeServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + for limit in [0, 1, 3] { + for pool in [None, Some(0), Some(1), Some(100)] { + let kind = pb::query_graph_request::Query::KgSearch(pb::KgSearchQuery { + query: "01ARZ3NDEKTSV4RRFFQ69G5FAV".into(), + kind: Some(Class::Memory.iri().into()), + pool, + }); + let expected = query_with_limit( + &local, + &local_repository, + Some(view("feature")), + kind.clone(), + limit, + ) + .await + .unwrap(); + let actual = client + .query_graph(pb::QueryGraphRequest { + repository: Some(repository.clone()), + view: Some(view("feature")), + query: Some(kind), + limit, + }) + .await + .unwrap() + .into_inner(); + assert_eq!(actual, expected, "limit={limit}, pool={pool:?}"); + assert_eq!(actual.nodes[0].id, memory_id); + assert_eq!(actual.nodes[0].types, memory.types); + } + } + drop(client); + stop.send(()).unwrap(); + server.await.unwrap(); +} From 571932c754f562c2dad122f94d752ee7eef60156 Mon Sep 17 00:00:00 2001 From: Bradley Hilton Date: Tue, 27 Oct 2026 11:34:46 +0100 Subject: [PATCH 5/5] perf: filter recorded queries early and bound match collection --- README.md | 15 +- atomic-cli/tests/query_view_filtering_test.rs | 70 ++++ atomic-core/src/output/repo/file.rs | 221 ++++++++++ atomic-core/src/output/repo/mod.rs | 5 +- atomic-repository/Cargo.toml | 2 + atomic-repository/src/content_search.rs | 326 ++++++++++++++- atomic-repository/src/lib.rs | 2 + atomic-repository/src/query_diagnostics.rs | 69 ++++ atomic-repository/src/repository/content.rs | 97 ++++- atomic-repository/src/repository/query.rs | 106 +++-- .../src/repository/tests/attribute_tests.rs | 15 + .../tests/conflict_surface_tests.rs | 15 + .../src/repository/tests/query_view_tests.rs | 389 +++++++++++++++++- .../src/repository/vault_triples.rs | 50 ++- .../tests/query_view_performance.rs | 116 ++++++ docs/query-view-performance.md | 111 +++++ libatomic/tests/query_view_service_test.rs | 154 +++++++ 17 files changed, 1694 insertions(+), 69 deletions(-) create mode 100644 atomic-repository/src/query_diagnostics.rs create mode 100644 atomic-repository/tests/query_view_performance.rs create mode 100644 docs/query-view-performance.md diff --git a/README.md b/README.md index 3fad0a21..c0d5b030 100644 --- a/README.md +++ b/README.md @@ -379,7 +379,20 @@ use request-scoped tree-sitter parsing, not persisted KG code edges. This correctness-first path renders/scans the selected view per query. RPC clients can select a view explicitly; omitting it uses the working copy's registered desired view. Other query verbs retain their existing scope semantics. The separate -working-copy content index is still available to its existing consumers. +working-copy content index is still available to its existing consumers. Existing +ambient ignore rules still suppress ignored paths, including recorded files; that +policy is intentional and unchanged. + +Recorded code search applies path/type filters **before reading file contents**, +then scans eligible files one at a time. It keeps at most the requested number of +best matches while still counting every match and directory facet—even with a +zero display limit. Facet-count ties sort by directory path. Rendered files over +10 MiB are skipped, not searched as truncated files. Closure, path-claim, and inode +attribute validation still precede request filtering: invalid global metadata +fails closed, while content-only errors in excluded files are deliberately unread. +Selected-file content errors still propagate. No persistent cache or reindex is +required. See [query resource diagnostics](docs/query-view-performance.md) for +memory bounds, costs excluded from those bounds, and a reproducible benchmark. For `query search`, `--kind` matches an exact, case-sensitive absolute class IRI against **any asserted type**, before candidate-pool truncation. An omitted or diff --git a/atomic-cli/tests/query_view_filtering_test.rs b/atomic-cli/tests/query_view_filtering_test.rs index 0c0dbac0..ed5d9519 100644 --- a/atomic-cli/tests/query_view_filtering_test.rs +++ b/atomic-cli/tests/query_view_filtering_test.rs @@ -142,6 +142,76 @@ fn query_verbs_render_filtered_json_and_human_output_over_the_service() { ); } +#[test] +fn code_keeps_exact_totals_and_late_source_ranking_at_small_limits() { + let dir = tempfile::tempdir().unwrap(); + let home = tempfile::tempdir().unwrap(); + let repo = Repository::init(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + for (path, content) in [ + ("docs/a.md", "needle\n".repeat(2500)), + ("src/z.rs", "prefix needle\r\nneedle\n".into()), + ] { + let full = dir.path().join(path); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(full, content).unwrap(); + repo.add(wc, path, TrackingOptions::default()).unwrap(); + } + record(&repo, "late ranked source fixture"); + let expected = repo + .search_content_on_view( + "dev", + "needle", + atomic_repository::ContentSearchOptions { + max_results: Some(1), + ..Default::default() + }, + ) + .unwrap(); + drop(repo); + let result = json( + dir.path(), + home.path(), + &["query", "code", "needle", "-n", "1", "--json"], + ); + assert_eq!(result, serde_json::to_value(expected).unwrap()); + assert_eq!(result["matches"][0]["path"], "src/z.rs"); + assert_eq!(result["total_matches"], 2502); + let result = json( + dir.path(), + home.path(), + &["query", "code", "needle", "-n", "0", "--json"], + ); + assert_eq!(result["matches"], serde_json::json!([])); + assert_eq!(result["total_matches"], 2502); + let narrow = json( + dir.path(), + home.path(), + &[ + "query", "code", "needle", "-g", "src/", "-t", "rs", "-n", "1", "--json", + ], + ); + assert_eq!(narrow["total_matches"], 2); + assert_eq!(narrow["matches"][0]["byte_offset"], 7); + let invalid = run( + dir.path(), + home.path(), + &["query", "code", "[", "-g", "missing/", "-n", "0"], + ); + assert!(!invalid.status.success()); + let human = run( + dir.path(), + home.path(), + &["query", "code", "needle", "-n", "1"], + ); + assert!(human.status.success()); + assert!(String::from_utf8_lossy(&human.stdout).contains("src/z.rs:1: prefix needle")); + assert!(!dir + .path() + .join(".atomic/content-index/manifest.json") + .exists()); +} + #[test] fn search_kind_with_tiny_pool_returns_the_matching_memory_over_the_service() { let dir = tempfile::tempdir().unwrap(); diff --git a/atomic-core/src/output/repo/file.rs b/atomic-core/src/output/repo/file.rs index 9a519e42..3bf57f49 100644 --- a/atomic-core/src/output/repo/file.rs +++ b/atomic-core/src/output/repo/file.rs @@ -983,12 +983,233 @@ where Ok((content, conflicts, had_fork_structure)) } +/// Render into a bounded consumer buffer using the ordinary semantic renderer. +/// +/// A missing content buffer means rendered output exceeded `max_bytes`; it is +/// never a partial file. Graph traversal and semantic-merge scratch are separate +/// from this consumer bound. Complete output matches the ordinary unbounded +/// renderer, including its conflict markers and semantic-merge resolution. +/// +/// # Examples +/// +/// ```no_run +/// use atomic_core::{ +/// change::ChangeStore, +/// output::{alive::RetrieveOptions, repo::{output_file_to_buffer_bounded, OutputError}}, +/// pristine::GraphTxnT, +/// types::{NodeId, Position}, +/// }; +/// fn preview( +/// txn: &T, changes: &C, position: Position, visibility: RetrieveOptions, +/// ) -> Result<(), OutputError> { +/// let output = output_file_to_buffer_bounded(txn, changes, position, visibility, 1024)?; +/// if let Some(bytes) = output.content { +/// assert!(bytes.len() <= 1024); +/// } // Otherwise the rendered file was too large; no truncated bytes escape. +/// Ok(()) +/// } +/// ``` +/// +/// # Errors +/// Propagates graph, content-store, and renderer failures. Only the buffer's own +/// limit rejection is classified as oversized content. +pub fn output_file_to_buffer_bounded( + txn: &T, + changes: &C, + position: Position, + retrieve_options: RetrieveOptions, + max_bytes: usize, +) -> Result { + let retrieved = retrieve_graph(txn, position, retrieve_options) + .map_err(|error| OutputError::Pristine(Box::new(error)))?; + let mut graph = retrieved.graph; + let order = compute_order(&mut graph); + let resolved = resolve_conflicts_semantically(txn, changes, &graph, &order)?; + let had_fork = !resolved.is_empty(); + let mut output = BoundedOutput { + writer: Writer::new(BoundedBytes { + bytes: Vec::new(), + max_bytes, + exceeded: false, + }), + }; + let result = output_graph_content_resolved( + changes, + |id| txn.get_external(id), + &graph, + &order, + &mut output, + &resolved, + ); + if output.writer.inner().exceeded { + return Ok(BoundedFileOutput { + content: None, + had_fork_structure: had_fork, + buffer_high_water: output.writer.inner().bytes.len(), + }); + } + result?; + let bytes = output.writer.into_inner().bytes; + Ok(BoundedFileOutput { + buffer_high_water: bytes.len(), + content: Some(bytes), + had_fork_structure: had_fork, + }) +} + +/// Bounded rendered bytes and consumer-buffer measurements, excluding renderer scratch. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct BoundedFileOutput { + /// `None` denotes oversized output; truncated content is never returned. + pub content: Option>, + pub had_fork_structure: bool, + /// Maximum retained output bytes before completion or the size rejection. + pub buffer_high_water: usize, +} + +struct BoundedBytes { + bytes: Vec, + max_bytes: usize, + exceeded: bool, +} + +impl BoundedBytes { + fn reject(&mut self) -> std::io::Error { + self.exceeded = true; + std::io::Error::other("rendered output exceeds consumer limit") + } +} + +impl Write for BoundedBytes { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.max_bytes.saturating_sub(self.bytes.len()) { + return Err(self.reject()); + } + let required = self.bytes.len() + bytes.len(); + if required > self.bytes.capacity() { + let capacity = self + .bytes + .capacity() + .saturating_mul(2) + .max(required) + .min(self.max_bytes); + self.bytes.reserve_exact(capacity - self.bytes.len()); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +struct BoundedOutput { + writer: Writer, +} + +impl crate::output::traits::VertexBuffer for BoundedOutput { + fn output_line( + &mut self, + node: crate::types::GraphNode, + get_contents: F, + ) -> Result<(), E> + where + E: From, + F: FnOnce(&mut [u8]) -> Result<(), E>, + { + // Check an emitted span before Writer allocates its scratch buffer. + // This is a rendered-span check, never a change-payload size estimate. + let sink = self.writer.inner_mut(); + if node.len() > sink.max_bytes.saturating_sub(sink.bytes.len()) { + return Err(sink.reject().into()); + } + self.writer.output_line(node, get_contents) + } + + fn output_conflict_marker( + &mut self, + marker: &str, + id: usize, + changes: Option<&[crate::types::Hash]>, + ) -> std::io::Result<()> { + self.writer.output_conflict_marker(marker, id, changes) + } +} + // TESTS #[cfg(test)] mod tests { use super::*; + #[test] + fn bounded_vertex_output_preserves_marker_bytes_and_stops_before_large_spans() { + use crate::output::traits::VertexBuffer; + use crate::types::{ChangePosition, GraphNode}; + fn emit(buffer: &mut impl VertexBuffer) -> std::io::Result<()> { + let node = GraphNode::new(NodeId::ROOT, ChangePosition::new(0), ChangePosition::new(2)); + buffer.output_line::(node, |bytes| { + bytes.copy_from_slice(b"aa"); + Ok(()) + })?; + buffer.begin_conflict(1, None)?; + buffer.output_line::(node, |bytes| { + bytes.copy_from_slice(b"bb"); + Ok(()) + })?; + buffer.conflict_next(1, None)?; + buffer.output_line::(node, |bytes| { + bytes.copy_from_slice(b"cc"); + Ok(()) + })?; + buffer.end_conflict(1) + } + let mut full = Writer::new(Vec::new()); + emit(&mut full).unwrap(); + let expected = full.into_inner(); + for cap in 0..=expected.len() + 1 { + let mut bounded = BoundedOutput { + writer: Writer::new(BoundedBytes { + bytes: Vec::new(), + max_bytes: cap, + exceeded: false, + }), + }; + let result = emit(&mut bounded); + assert!(bounded.writer.inner().bytes.len() <= cap); + assert!(bounded.writer.inner().bytes.capacity() <= cap); + assert_eq!( + bounded.writer.inner().exceeded, + cap < expected.len(), + "cap={cap}" + ); + if cap >= expected.len() { + result.unwrap(); + assert_eq!(bounded.writer.into_inner().bytes, expected); + } else { + assert!(result.is_err()); + } + } + let mut bounded = BoundedOutput { + writer: Writer::new(BoundedBytes { + bytes: Vec::new(), + max_bytes: 10, + exceeded: false, + }), + }; + let huge = GraphNode::new( + NodeId::ROOT, + ChangePosition::new(0), + ChangePosition::new(100_000_000), + ); + assert!(bounded + .output_line::(huge, |_| panic!("oversized span must not be loaded")) + .is_err()); + assert!(bounded.writer.inner().bytes.is_empty()); + } + // FileOutputOptions Tests #[test] diff --git a/atomic-core/src/output/repo/mod.rs b/atomic-core/src/output/repo/mod.rs index 31ab9057..0db7fa8c 100644 --- a/atomic-core/src/output/repo/mod.rs +++ b/atomic-core/src/output/repo/mod.rs @@ -119,8 +119,9 @@ pub use content::{ }; pub use error::{OutputError, OutputResult}; pub use file::{ - output_file, output_file_to_buffer, output_file_to_buffer_with_options, FileOutputError, - FileOutputOptions, FileOutputResult, + output_file, output_file_to_buffer, output_file_to_buffer_bounded, + output_file_to_buffer_with_options, BoundedFileOutput, FileOutputError, FileOutputOptions, + FileOutputResult, }; pub use names::{live_inode_names, live_inode_names_with_claims}; pub use options::OutputOptions; diff --git a/atomic-repository/Cargo.toml b/atomic-repository/Cargo.toml index 0fec25c4..51531dc0 100644 --- a/atomic-repository/Cargo.toml +++ b/atomic-repository/Cargo.toml @@ -15,6 +15,8 @@ rust-version.workspace = true # `ATOMIC_INJECT_WALK_READ_DIR_ERROR` walk-error injection). Shipping builds # never compile these seams: no environment value can alter adoption. adoption-test-injection = [] +# Request-local deterministic resource counters for query tests/benchmarks only. +query-test-instrumentation = [] [dependencies] unicode-normalization = "0.1" diff --git a/atomic-repository/src/content_search.rs b/atomic-repository/src/content_search.rs index 0ad0ea67..3d7a9616 100644 --- a/atomic-repository/src/content_search.rs +++ b/atomic-repository/src/content_search.rs @@ -179,7 +179,7 @@ where /// /// Results are ranked: source files (`src/`, `lib/`, `pkg/`, `internal/`) /// are prioritized over config, test, build, and generated files. Within -/// each tier, results preserve filesystem order from the index. +/// each tier, results follow path order and preserve the verifier's within-file order. /// /// The returned [`ContentSearchResult`] includes both the ranked matches /// (capped to `max_results`) and the `total_matches` count so callers @@ -228,9 +228,133 @@ pub fn search_content( Ok(rank_content_matches(raw_matches, display_limit)) } +/// Recorded-query byte and regex-automaton cutoff (10 MiB). +pub(crate) const RECORDED_CONTENT_LIMIT: usize = 10 * 1024 * 1024; + +/// An immutable path/filter plan and bounded collector for recorded content. +/// Paths are scanned in result order (tier, then canonical path), so retaining +/// the first N records is equivalent to the exhaustive stable tier sort. +pub(crate) struct RecordedContentSearch { + re: regex::bytes::Regex, + paths: Vec, + display_limit: usize, +} + +impl RecordedContentSearch { + pub(crate) fn new<'path>( + paths: impl IntoIterator, + pattern: &str, + opts: ContentSearchOptions, + ) -> Result { + // Compile even for empty scopes and zero display limits. + let re = regex::bytes::RegexBuilder::new(pattern) + .case_insensitive(opts.case_insensitive) + .size_limit(RECORDED_CONTENT_LIMIT) + .dfa_size_limit(RECORDED_CONTENT_LIMIT) + .build() + .map_err(|error| ContentSearchError::InvalidPattern(error.to_string()))?; + let mut paths: Vec<_> = paths.into_iter().map(std::path::PathBuf::from).collect(); + paths.sort(); + let index = syntext::path::PathIndex::build(&paths); + let filter = syntext::path::filter::build_filter( + &index, + opts.file_type.as_deref(), + opts.exclude_type.as_deref(), + opts.path_filter.as_deref(), + ); + let mut paths: Vec<_> = paths + .into_iter() + .enumerate() + .filter(|(id, _)| { + filter + .as_ref() + .is_none_or(|filter| filter.file_ids.contains(*id as u32)) + }) + .map(|(_, path)| path.to_string_lossy().into_owned()) + .collect(); + paths.sort_by(|a, b| recorded_path_order(a, b)); + Ok(Self { + re, + paths, + display_limit: opts.max_results.unwrap_or(50), + }) + } + + pub(crate) fn run( + self, + mut retrieve: impl FnMut(&str) -> Result>, E>, + ) -> Result { + let mut matches = Vec::new(); + let mut total_matches = 0; + let mut directories = std::collections::BTreeMap::::new(); + for path in self.paths { + let Some(content) = retrieve(&path)? else { + continue; + }; + if content.len() > RECORDED_CONTENT_LIMIT || syntext::index::is_binary(&content) { + continue; + } + let mut line_start = 0; + let mut file_count = 0; + // This borrowed-line adapter implements syntext 1.4's verifier + // contract: first match per line, CR stripped only before LF, no + // artificial trailing empty line, and file-level binary detection. + // Differential tests below keep it pinned to verify_regex, including + // NULs beyond the first 8192 bytes. No dependency fork is required. + for (index, raw_line) in content.split_inclusive(|byte| *byte == b'\n').enumerate() { + let line = raw_line + .strip_suffix(b"\n") + .map(|line| line.strip_suffix(b"\r").unwrap_or(line)) + .unwrap_or(raw_line); + if let Some(found) = self.re.find(line) { + file_count += 1; + if matches.len() < self.display_limit { + matches.push(ContentMatch { + path: path.clone(), + line_number: (index + 1) as u32, + line_content: String::from_utf8_lossy(line).into_owned(), + byte_offset: (line_start + found.start()) as u64, + submatch_start: found.start(), + submatch_end: found.end(), + }); + #[cfg(any(test, feature = "query-test-instrumentation"))] + crate::query_diagnostics::retained(matches.len()); + } + } + line_start += raw_line.len(); + } + if file_count > 0 { + total_matches += file_count; + *directories.entry(directory_bucket(&path)).or_default() += file_count; + } + } + Ok(ContentSearchResult { + matches, + total_matches, + dir_facets: directory_facets(directories), + }) + } +} + +fn recorded_path_order(a: &str, b: &str) -> std::cmp::Ordering { + path_tier(a).cmp(&path_tier(b)).then_with(|| a.cmp(b)) +} + +fn directory_bucket(path: &str) -> String { + path.splitn(3, '/').take(2).collect::>().join("/") +} + +fn directory_facets(counts: impl IntoIterator) -> Vec<(String, usize)> { + let mut facets: Vec<_> = counts.into_iter().collect(); + facets.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + facets.truncate(10); + facets +} + /// Search graph-rendered bytes without consulting the working-copy index. /// Path/type matching and line verification use the same syntext machinery as /// indexed search; filtering and ranking happen before the display limit. +#[cfg(test)] pub(crate) fn search_file_contents( files: &std::collections::BTreeMap>, pattern: &str, @@ -276,36 +400,30 @@ fn rank_content_matches( display_limit: usize, ) -> ContentSearchResult { let total_matches = raw_matches.len(); + #[cfg(any(test, feature = "query-test-instrumentation"))] + crate::query_diagnostics::retained(total_matches); // Build per-directory match counts for the facets summary. let mut dir_counts: std::collections::HashMap = std::collections::HashMap::new(); for m in &raw_matches { - let path_str = m.path.to_string_lossy(); - // Use the first two path components as the directory bucket. - let dir = path_str - .splitn(3, '/') - .take(2) - .collect::>() - .join("/"); + let dir = directory_bucket(&m.path.to_string_lossy()); *dir_counts.entry(dir).or_insert(0) += 1; } - // Sort directory facets by count (descending). - let mut dir_facets: Vec<(String, usize)> = dir_counts.into_iter().collect(); - dir_facets.sort_by_key(|entry| std::cmp::Reverse(entry.1)); - dir_facets.truncate(10); // top 10 directories + let dir_facets = directory_facets(dir_counts); // Rank matches: source code files first, then everything else. - let mut ranked: Vec<(usize, &syntext::SearchMatch)> = raw_matches - .iter() - .map(|m| (path_tier(m.path.to_string_lossy().as_ref()), m)) - .collect(); - ranked.sort_by_key(|(tier, _)| *tier); + let mut ranked: Vec<_> = raw_matches.iter().collect(); + // syntext's indexed results and the recorded oracle already arrive in path + // order. Use the same stable comparator as the bounded path plan; records + // within each file retain the verifier's original order. + ranked + .sort_by(|a, b| recorded_path_order(&a.path.to_string_lossy(), &b.path.to_string_lossy())); let matches: Vec = ranked .into_iter() .take(display_limit) - .map(|(_, m)| ContentMatch { + .map(|m| ContentMatch { path: m.path.to_string_lossy().to_string(), line_number: m.line_number, line_content: String::from_utf8_lossy(&m.line_content).to_string(), @@ -456,7 +574,8 @@ pub struct ContentSearchOptions { /// Exclude files of this type. pub exclude_type: Option, /// Maximum number of results to display (default: 50). - /// Internally, more results are fetched for ranking. + /// Recorded search counts all eligible matches while retaining at most this + /// many records. Working-copy indexed search retains its candidate overfetch. pub max_results: Option, /// Case-insensitive matching. pub case_insensitive: bool, @@ -586,6 +705,175 @@ impl From for ContentSearchError { mod tests { use super::*; + fn streaming( + files: &std::collections::BTreeMap>, + pattern: &str, + opts: ContentSearchOptions, + ) -> ContentSearchResult { + RecordedContentSearch::new(files.keys().map(String::as_str), pattern, opts) + .unwrap() + .run::(|path| Ok(Some(files[path].clone()))) + .unwrap() + } + + #[test] + fn recorded_stream_matches_exhaustive_verifier_boundaries_and_filters() { + let files = std::collections::BTreeMap::from([ + ( + "src/z.rs".into(), + b"one needle needle\r\n\r\nNEEDLE\nlast\r".to_vec(), + ), + ("src/a.rs".into(), b"\n\nneedle\n".to_vec()), + ("tests/a.rs".into(), b"needle\n".to_vec()), + ("src/empty.rs".into(), Vec::new()), + ("src/binary.rs".into(), b"needle\0\n".to_vec()), + ("src/invalid.rs".into(), b"before\xffneedle\n".to_vec()), + ( + "src/unicode.rs".into(), + "é needle🦀\r\n終わり needle".as_bytes().to_vec(), + ), + ("notes.md".into(), b"needle\n".to_vec()), + ( + "src/late-nul.rs".into(), + [vec![b'x'; 8200], b"\nneedle\0\n".to_vec()].concat(), + ), + ]); + for pattern in [ + "needle", + "", + "^", + "$", + "^$", + "needle.*", + r"(?-u)\xFF|\x00", + "needle\\n", + ] { + for max_results in [None, Some(0), Some(1), Some(5), Some(100)] { + for (path_filter, file_type, exclude_type) in [ + (None, None, None), + (Some("src/"), Some("rs"), None), + (Some("**/*.rs"), None, Some("rs")), + (Some("notes.md"), None, None), + (Some("missing/**"), Some("unknown_type"), None), + (None, Some("unknown_type"), None), + (Some("."), None, None), + (Some("/src/"), None, None), + (Some("../src/"), None, None), + ] { + let opts = ContentSearchOptions { + max_results, + case_insensitive: true, + path_filter: path_filter.map(str::to_owned), + file_type: file_type.map(str::to_owned), + exclude_type: exclude_type.map(str::to_owned), + }; + let expected = search_file_contents(&files, pattern, opts.clone()).unwrap(); + let actual = streaming(&files, pattern, opts); + assert_eq!( + serde_json::to_value(actual).unwrap(), + serde_json::to_value(expected).unwrap(), + "pattern={pattern:?} limit={max_results:?} filter={path_filter:?}" + ); + } + } + } + } + + #[test] + fn randomized_recorded_stream_matches_exhaustive_order_counts_and_offsets() { + use rand::{Rng, SeedableRng}; + let mut rng = rand::rngs::StdRng::seed_from_u64(11); + for case in 0..80 { + let mut files = std::collections::BTreeMap::new(); + for path in ["docs/a.md", "src/z.rs", "src/a.rs", "tests/b.rs", "root.py"] { + let mut bytes: Vec = (0..rng.gen_range(0..500)) + .map(|_| { + let alphabet = b"aaaazz\r\n\xff "; + alphabet[rng.gen_range(0..alphabet.len())] + }) + .collect(); + if case % 7 == 0 && !bytes.is_empty() { + bytes[0] = 0; + } + files.insert(path.into(), bytes); + } + for pattern in ["a+", "z|a", "^$", "", r"(?-u)\xFF"] { + for max_results in [Some(0), Some(1), Some(10), None] { + let opts = ContentSearchOptions { + max_results, + ..Default::default() + }; + assert_eq!( + serde_json::to_value(streaming(&files, pattern, opts.clone())).unwrap(), + serde_json::to_value(search_file_contents(&files, pattern, opts).unwrap()) + .unwrap(), + "case={case} pattern={pattern}" + ); + } + } + } + } + + #[test] + fn recorded_stream_counts_dense_matches_without_retaining_them() { + let files = std::collections::BTreeMap::from([( + "src/dense.rs".into(), + b"needle\n".repeat(100_000), + )]); + for limit in [0, 1, 10] { + let (result, resources) = crate::query_diagnostics::measure(|| { + streaming( + &files, + "needle", + ContentSearchOptions { + max_results: Some(limit), + ..Default::default() + }, + ) + }); + assert_eq!(result.total_matches, 100_000); + assert_eq!(result.matches.len(), limit); + assert_eq!(result.dir_facets, [("src/dense.rs".into(), 100_000)]); + assert_eq!(resources.displayed_records_high_water, limit); + } + for paths in [vec![], vec!["missing.rs"]] { + assert!(RecordedContentSearch::new( + paths, + "[", + ContentSearchOptions { + max_results: Some(0), + ..Default::default() + } + ) + .is_err()); + } + let tied: std::collections::BTreeMap<_, _> = (0..12) + .rev() + .map(|i| (format!("dir{i:02}/a.rs"), b"needle\n".to_vec())) + .collect(); + let result = streaming(&tied, "needle", ContentSearchOptions::default()); + assert_eq!( + result.dir_facets, + (0..10) + .map(|i| (format!("dir{i:02}/a.rs"), 1)) + .collect::>() + ); + } + + #[test] + fn recorded_stream_applies_the_exact_content_cutoff() { + let files = std::collections::BTreeMap::from([ + ("src/exact.rs".into(), vec![b'x'; RECORDED_CONTENT_LIMIT]), + ( + "src/oversized.rs".into(), + vec![b'x'; RECORDED_CONTENT_LIMIT + 1], + ), + ]); + let result = streaming(&files, "^x", ContentSearchOptions::default()); + assert_eq!(result.total_matches, 1); + assert_eq!(result.matches[0].path, "src/exact.rs"); + } + #[test] fn search_content_excludes_ignored_and_internal_paths() { let dir = tempfile::tempdir().unwrap(); diff --git a/atomic-repository/src/lib.rs b/atomic-repository/src/lib.rs index 4d243d7f..bd53fd34 100644 --- a/atomic-repository/src/lib.rs +++ b/atomic-repository/src/lib.rs @@ -130,6 +130,8 @@ pub mod ai; // Content search powered by syntext pub mod content_search; +#[cfg(any(test, feature = "query-test-instrumentation"))] +pub mod query_diagnostics; // OCI image layout production (sandbox stage/seal) pub mod oci; diff --git a/atomic-repository/src/query_diagnostics.rs b/atomic-repository/src/query_diagnostics.rs new file mode 100644 index 00000000..3acaedab --- /dev/null +++ b/atomic-repository/src/query_diagnostics.rs @@ -0,0 +1,69 @@ +//! Opt-in, thread-local query resource measurements for tests and benchmarks. +//! +//! These counters describe consumer buffers, not ChangeStore caches, renderer +//! span buffers, graph metadata, or semantic-merge scratch. Measure synchronous +//! repository calls on the calling thread; do not nest measurement scopes. + +use std::cell::RefCell; + +/// Consumer resource high-water marks for a synchronous recorded query. +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)] +pub struct QueryResources { + pub files_retrieved: usize, + pub bytes_retained: usize, + pub output_buffer_high_water: usize, + pub displayed_records_high_water: usize, +} + +thread_local! { + static RESOURCES: RefCell> = const { RefCell::new(None) }; +} + +/// Run one synchronous query and collect its consumer resource counters. +/// +/// # Examples +/// +/// ``` +/// use atomic_repository::query_diagnostics::measure; +/// let (answer, resources) = measure(|| 42); +/// assert_eq!(answer, 42); +/// assert_eq!(resources.files_retrieved, 0); +/// ``` +/// +/// # Panics +/// Panics if measurements are nested on the same thread. +pub fn measure(query: impl FnOnce() -> T) -> (T, QueryResources) { + struct Reset; + impl Drop for Reset { + fn drop(&mut self) { + RESOURCES.with(|slot| *slot.borrow_mut() = None); + } + } + RESOURCES.with(|slot| { + let mut slot = slot.borrow_mut(); + assert!(slot.is_none(), "query measurements cannot be nested"); + *slot = Some(QueryResources::default()); + }); + let _reset = Reset; + let result = query(); + let stats = RESOURCES.with(|slot| slot.borrow_mut().take().expect("measurement is active")); + (result, stats) +} + +pub(crate) fn retrieved(bytes: usize) { + RESOURCES.with(|slot| { + if let Some(stats) = slot.borrow_mut().as_mut() { + stats.files_retrieved += 1; + stats.bytes_retained += bytes; + stats.output_buffer_high_water = stats.output_buffer_high_water.max(bytes); + } + }); +} + +pub(crate) fn retained(count: usize) { + RESOURCES.with(|slot| { + if let Some(stats) = slot.borrow_mut().as_mut() { + stats.displayed_records_high_water = stats.displayed_records_high_water.max(count); + } + }); +} diff --git a/atomic-repository/src/repository/content.rs b/atomic-repository/src/repository/content.rs index ba5af4ab..2b226239 100644 --- a/atomic-repository/src/repository/content.rs +++ b/atomic-repository/src/repository/content.rs @@ -1065,6 +1065,75 @@ fn try_retrieve_linear_content_with_filter( position: Position, options: &atomic_core::output::alive::RetrieveOptions, ) -> atomic_core::record::RecordResult>> +where + T: atomic_core::pristine::GraphTxnT + + atomic_core::pristine::InodeGraphOps + + atomic_core::pristine::TreeTxnT, + C: atomic_core::change::ChangeStore, +{ + Ok( + try_retrieve_linear_content_bounded(txn, changes, inode, position, options, usize::MAX)? + .and_then(|content| match content { + BoundedContent::Complete(bytes) => Some(bytes), + BoundedContent::Oversized { .. } => None, + }), + ) +} + +#[derive(Debug)] +pub(crate) enum BoundedContent { + Complete(Vec), + Oversized { retained_bytes: usize }, +} + +/// The query consumer's cap applies to emitted bytes, not to raw change sizes. +/// Both paths use the ordinary graph visibility and semantic rendering rules. +pub(crate) fn retrieve_content_bounded( + txn: &T, + changes: &C, + inode: Inode, + position: Position, + options: atomic_core::output::alive::RetrieveOptions, + max_bytes: usize, +) -> atomic_core::record::RecordResult +where + T: atomic_core::pristine::GraphTxnT + + atomic_core::pristine::InodeGraphOps + + atomic_core::pristine::TreeTxnT, + C: atomic_core::change::ChangeStore, +{ + if let Some(content) = + try_retrieve_linear_content_bounded(txn, changes, inode, position, &options, max_bytes)? + { + return Ok(content); + } + let result = match atomic_core::pristine::InodeScopedGraph::new(txn, inode) { + Ok(scoped) => atomic_core::output::repo::output_file_to_buffer_bounded( + &scoped, changes, position, options, max_bytes, + ), + Err(_) => atomic_core::output::repo::output_file_to_buffer_bounded( + txn, changes, position, options, max_bytes, + ), + } + .map_err(|error| { + atomic_core::record::RecordError::Io(std::io::Error::other(error.to_string())) + })?; + Ok(match result.content { + Some(bytes) => BoundedContent::Complete(bytes), + None => BoundedContent::Oversized { + retained_bytes: result.buffer_high_water, + }, + }) +} + +fn try_retrieve_linear_content_bounded( + txn: &T, + changes: &C, + inode: Inode, + position: Position, + options: &atomic_core::output::alive::RetrieveOptions, + max_bytes: usize, +) -> atomic_core::record::RecordResult> where T: atomic_core::pristine::GraphTxnT + atomic_core::pristine::InodeGraphOps @@ -1076,7 +1145,7 @@ where let trace_retrieve = std::env::var_os("ATOMIC_TRACE_RETRIEVE").is_some(); if position == Position::ROOT { - return Ok(Some(Vec::new())); + return Ok(Some(BoundedContent::Complete(Vec::new()))); } let inode_marker = position.inode_node(); @@ -1139,9 +1208,28 @@ where } .into()); } - let mut span = vec![0u8; end - start]; + let length = end - start; + if length > max_bytes.saturating_sub(content.len()) { + return Ok(Some(BoundedContent::Oversized { + retained_bytes: content.len(), + })); + } + let offset = content.len(); + if offset + length > content.capacity() { + let capacity = content + .capacity() + .saturating_mul(2) + .max(offset + length) + .min(max_bytes); + content.reserve_exact(capacity - offset); + } + content.resize(offset + length, 0); let copied = changes - .get_contents(|id| txn.get_external(id).ok().flatten(), node, &mut span) + .get_contents( + |id| txn.get_external(id).ok().flatten(), + node, + &mut content[offset..], + ) .map_err(|error| { atomic_core::record::RecordError::Io(std::io::Error::other(format!( "content span {}..{} for {} exceeds change {} content length: {}", @@ -1162,10 +1250,9 @@ where } .into()); } - content.extend_from_slice(&span); } - Ok(Some(content)) + Ok(Some(BoundedContent::Complete(content))) } #[cfg(test)] diff --git a/atomic-repository/src/repository/query.rs b/atomic-repository/src/repository/query.rs index c7bf9859..dc40eb99 100644 --- a/atomic-repository/src/repository/query.rs +++ b/atomic-repository/src/repository/query.rs @@ -10,20 +10,30 @@ use atomic_core::pristine::KgTxnT; use atomic_core::KgNode; use super::*; -use crate::content_search::{search_file_contents, ContentSearchOptions, ContentSearchResult}; +use crate::content_search::{ + ContentSearchOptions, ContentSearchResult, RecordedContentSearch, RECORDED_CONTENT_LIMIT, +}; +use atomic_core::pristine::GraphVisibilityClosure; -pub(super) struct QueryView { - pub files: BTreeMap>, +pub(super) struct QueryView<'repo> { + repo: &'repo Repository, + pub(super) txn: atomic_core::ReadTxn, + visibility: GraphVisibilityClosure, + files: BTreeMap)>, changes: HashSet, views: HashSet, nodes: BTreeMap, } -impl QueryView { +impl QueryView<'_> { pub fn nodes(&self) -> &BTreeMap { &self.nodes } + pub fn contains_file(&self, path: &str) -> bool { + self.files.contains_key(path) + } + /// Project ambient candidates before ranking. File identities are canonical, /// but their presence must come from the selected view's current projection. pub fn project_node( @@ -68,16 +78,58 @@ impl QueryView { pattern: &str, opts: ContentSearchOptions, ) -> Result { - search_file_contents(&self.files, pattern, opts).map_err(|error| { + let plan = self.content_plan(pattern, opts).map_err(|error| { RepositoryError::InvalidOperation { message: format!("code search failed: {error}"), } + })?; + self.search_content_plan(plan) + } + + pub fn content_plan( + &self, + pattern: &str, + opts: ContentSearchOptions, + ) -> Result { + RecordedContentSearch::new(self.files.keys().map(String::as_str), pattern, opts) + } + + pub fn search_content_plan( + &self, + plan: RecordedContentSearch, + ) -> Result { + plan.run(|path| { + let (inode, position) = self.files[path]; + let rendered = super::content::retrieve_content_bounded( + &self.txn, + &self.repo.change_store, + inode, + position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(self.visibility.clone()), + RECORDED_CONTENT_LIMIT, + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + let (bytes, retained_bytes) = match rendered { + super::content::BoundedContent::Complete(bytes) => { + let retained = bytes.len(); + (Some(bytes), retained) + } + super::content::BoundedContent::Oversized { retained_bytes } => { + (None, retained_bytes) + } + }; + #[cfg(any(test, feature = "query-test-instrumentation"))] + crate::query_diagnostics::retrieved(retained_bytes); + #[cfg(not(any(test, feature = "query-test-instrumentation")))] + let _ = retained_bytes; + Ok(bytes) }) } } impl Repository { - pub(super) fn query_view(&self, view_name: &str) -> Result { + pub(super) fn query_view(&self, view_name: &str) -> Result, RepositoryError> { let txn = self.pristine.read_txn()?; let view = txn .get_view(view_name)? @@ -131,16 +183,7 @@ impl Repository { if attrs.materialization.kind != atomic_core::change::InodeKind::Regular { continue; } - let bytes = super::content::retrieve_content_with_filter_fast( - &txn, - &self.change_store, - item.inode, - item.position, - atomic_core::output::alive::RetrieveOptions::new() - .with_graph_visibility(visibility.clone()), - ) - .map_err(|error| RepositoryError::Database(error.to_string()))?; - files.insert(path, bytes); + files.insert(path, (item.inode, item.position)); } let nodes = files .keys() @@ -153,6 +196,9 @@ impl Repository { }) .collect(); Ok(QueryView { + repo: self, + txn, + visibility, files, changes, views, @@ -163,6 +209,8 @@ impl Repository { /// Search recorded source bytes through a view's validated ancestor and /// dependency closure. Unrecorded files/edits and the working-copy content /// index are excluded; no index rebuild or checkout occurs. + /// Existing ambient ignore rules still apply. Totals/facets are exact even + /// with zero displayed matches; rendered files over 10 MiB are skipped. pub fn search_content_on_view( &self, view_name: &str, @@ -174,6 +222,8 @@ impl Repository { /// Like [`Self::search_content_on_view`], restricted to a union of exact /// repository-relative paths and directory prefixes before matching. + /// Path/type filters precede content I/O; closure/path/attribute validation + /// remains global. Content errors outside the selected files are not read. pub fn search_content_paths_on_view( &self, view_name: &str, @@ -205,15 +255,21 @@ impl Repository { target: &str, ) -> Result, RepositoryError> { let scope = self.query_view(view_name)?; - let sources: Vec<_> = scope - .files - .iter() - .filter_map(|(path, bytes)| { - std::str::from_utf8(bytes) - .ok() - .map(|source| (path.clone(), source.to_owned())) - }) - .collect(); + let mut sources = Vec::new(); + for (path, (inode, position)) in &scope.files { + let bytes = super::content::retrieve_content_with_filter_fast( + &scope.txn, + &self.change_store, + *inode, + *position, + atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(scope.visibility.clone()), + ) + .map_err(|error| RepositoryError::Database(error.to_string()))?; + if let Ok(source) = String::from_utf8(bytes) { + sources.push((path.clone(), source)); + } + } Ok(SyntaxIndex::from_sources(&sources).callers(target)) } } diff --git a/atomic-repository/src/repository/tests/attribute_tests.rs b/atomic-repository/src/repository/tests/attribute_tests.rs index 038b2b9d..c7d2103e 100644 --- a/atomic-repository/src/repository/tests/attribute_tests.rs +++ b/atomic-repository/src/repository/tests/attribute_tests.rs @@ -117,6 +117,21 @@ fn concurrent_attribute_values_surface_as_status_conflict() { .unwrap(); assert_eq!(entry.status(), FileStatus::Conflicted); assert_eq!(entry.details(), Some("inode attribute conflict")); + let error = repo + .search_content_paths_on_view( + "dev", + "content", + crate::ContentSearchOptions { + max_results: Some(0), + ..Default::default() + }, + &["elsewhere.rs".into()], + ) + .unwrap_err(); + assert!( + error.to_string().contains("conflicting inode attributes"), + "{error}" + ); } #[cfg(unix)] diff --git a/atomic-repository/src/repository/tests/conflict_surface_tests.rs b/atomic-repository/src/repository/tests/conflict_surface_tests.rs index 8fc6af2f..03f796f5 100644 --- a/atomic-repository/src/repository/tests/conflict_surface_tests.rs +++ b/atomic-repository/src/repository/tests/conflict_surface_tests.rs @@ -195,6 +195,21 @@ fn test_name_conflict_same_path_creates_are_surfaced() { .any(|path| path == "new.txt"), "status must surface PATH_CLAIMS conflict before materialization" ); + let error = repo + .search_content_paths_on_view( + "dev", + "seed", + crate::ContentSearchOptions { + max_results: Some(0), + ..Default::default() + }, + &["seed.txt".into()], + ) + .unwrap_err(); + assert!( + error.to_string().contains("unresolved path claims"), + "{error}" + ); // Synthesized marker files are evidence, not repository objects. They must // bypass both ident expansion and external smudge drivers. diff --git a/atomic-repository/src/repository/tests/query_view_tests.rs b/atomic-repository/src/repository/tests/query_view_tests.rs index 6f401349..a1b70cca 100644 --- a/atomic-repository/src/repository/tests/query_view_tests.rs +++ b/atomic-repository/src/repository/tests/query_view_tests.rs @@ -1,9 +1,386 @@ use super::*; -use crate::{ContentSearchOptions, RecordOptions}; +use crate::{ContentSearchOptions, CrossViewInsertOptions, RecordOptions}; use atomic_core::kg::{iri, ontology::Class}; use atomic_core::pristine::vault::KgNode; use atomic_core::pristine::{KgMutTxnT, ViewTxnT}; +#[test] +fn narrow_query_retrieves_only_selected_bytes_and_retains_only_display_limit() { + let (temp, repo) = create_temp_repo(); + for (path, content) in [ + ("src/selected.rs", "needle\n".repeat(100)), + ("src/other.rs", "needle\n".repeat(100)), + ("src-other/other.rs", "needle\n".repeat(100)), + ("src/notes.md", "needle\n".repeat(100)), + ] { + let full = temp.path().join(path); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(full, content).unwrap(); + repo.add(path, TrackingOptions::default()).unwrap(); + } + record(&repo, "record selective query fixture"); + let (result, resources) = crate::query_diagnostics::measure(|| { + repo.search_content_paths_on_view( + "dev", + "needle", + ContentSearchOptions { + max_results: Some(1), + ..Default::default() + }, + &["src/selected.rs".into()], + ) + .unwrap() + }); + assert_eq!(result.total_matches, 100); + assert_eq!(result.matches.len(), 1); + assert_eq!(resources.files_retrieved, 1, "{resources:?}"); + assert!(resources.displayed_records_high_water <= 1, "{resources:?}"); + + let (scope, resources) = crate::query_diagnostics::measure(|| repo.query_view("dev").unwrap()); + assert_eq!( + resources.files_retrieved, 0, + "metadata projection must be lazy" + ); + assert_eq!(scope.nodes().len(), 4); + drop(scope); + for (paths, glob, kind, exclude, expected) in [ + (vec!["src/"], None, None, None, 3), + ( + vec!["src", "src/", "src/selected.rs"], + Some("**/*.rs"), + Some("rs"), + None, + 2, + ), + (vec!["src"], None, None, Some("rs"), 1), + (vec!["src/selected.rs"], Some("**/*.md"), None, None, 0), + (vec!["src/selected"], None, None, None, 0), + (vec!["missing"], None, None, None, 0), + (vec!["."], None, None, None, 0), + (vec!["/src"], None, None, None, 0), + (vec!["../src"], None, None, None, 0), + ] { + let (result, resources) = crate::query_diagnostics::measure(|| { + repo.search_content_paths_on_view( + "dev", + "needle", + ContentSearchOptions { + path_filter: glob.map(str::to_owned), + file_type: kind.map(str::to_owned), + exclude_type: exclude.map(str::to_owned), + max_results: Some(0), + ..Default::default() + }, + &paths + .iter() + .map(|path| (*path).to_owned()) + .collect::>(), + ) + .unwrap() + }); + assert_eq!(result.total_matches, expected * 100, "paths={paths:?}"); + assert_eq!(resources.files_retrieved, expected, "paths={paths:?}"); + assert_eq!(resources.displayed_records_high_water, 0); + } + std::fs::write(temp.path().join(".atomicignore"), "src/selected.rs\n").unwrap(); + let (result, resources) = crate::query_diagnostics::measure(|| { + repo.search_content_paths_on_view( + "dev", + "needle", + ContentSearchOptions::default(), + &["src/selected.rs".into()], + ) + .unwrap() + }); + assert_eq!( + result.total_matches, 0, + "existing ambient ignore behavior is intentional" + ); + assert_eq!(resources.files_retrieved, 0); +} + +#[test] +fn selected_content_errors_propagate_but_excluded_content_is_not_read() { + let (temp, repo) = create_temp_repo(); + std::fs::write(temp.path().join("good.rs"), "needle\n").unwrap(); + repo.add("good.rs", TrackingOptions::default()).unwrap(); + record(&repo, "good source"); + std::fs::write(temp.path().join("bad.rs"), "other\n").unwrap(); + repo.add("bad.rs", TrackingOptions::default()).unwrap(); + let bad = record(&repo, "bad source"); + // Reopen to eliminate cached payloads before corrupting a disposable store. + drop(repo); + let repo = Repository::open(temp.path()).unwrap(); + std::fs::write( + repo.change_store().change_path(&bad), + b"invalid stored payload", + ) + .unwrap(); + let selected = |path: &str, pattern: &str, limit| { + repo.search_content_paths_on_view( + "dev", + pattern, + ContentSearchOptions { + max_results: Some(limit), + ..Default::default() + }, + &[path.into()], + ) + }; + assert_eq!(selected("good.rs", "needle", 1).unwrap().total_matches, 1); + assert!(selected("bad.rs", "other", 1).is_err()); + assert!(selected("bad.rs", "other", 0).is_err()); + assert!(selected("missing", "[", 0).is_err()); + assert!(repo + .vault_kg_search_on_view("dev", "other", 10, None) + .is_err()); +} + +#[test] +fn recorded_query_plan_keeps_one_snapshot_across_later_records() { + let (temp, repo) = create_temp_repo(); + std::fs::write(temp.path().join("source.rs"), "before_marker\n").unwrap(); + repo.add("source.rs", TrackingOptions::default()).unwrap(); + record(&repo, "before snapshot"); + let scope = repo.query_view("dev").unwrap(); + std::fs::write(temp.path().join("source.rs"), "after_marker\n").unwrap(); + record(&repo, "after snapshot"); + let old = scope + .search_content( + "before_marker|after_marker", + ContentSearchOptions::default(), + ) + .unwrap(); + assert_eq!(old.matches[0].line_content, "before_marker"); + let new = repo + .search_content_on_view( + "dev", + "before_marker|after_marker", + ContentSearchOptions::default(), + ) + .unwrap(); + assert_eq!(new.matches[0].line_content, "after_marker"); +} + +#[test] +fn bounded_retrieval_matches_full_rendering_for_linear_and_deleted_graphs() { + use super::super::content::{retrieve_content_bounded, BoundedContent}; + let (temp, mut repo) = create_temp_repo(); + std::fs::write(temp.path().join("source.rs"), "first\nsecond\nthird\n").unwrap(); + repo.add("source.rs", TrackingOptions::default()).unwrap(); + record(&repo, "linear source"); + for revision in ["first\nsecond\nthird\n", "first\nthird\n"] { + if revision == "first\nthird\n" { + std::fs::write(temp.path().join("source.rs"), revision).unwrap(); + record(&repo, "delete middle line"); + } + let txn = repo.pristine().read_txn().unwrap(); + let inode = txn.get_inode("source.rs").unwrap().unwrap(); + let position = txn.inode_position(inode).unwrap().unwrap(); + let view = txn.get_view("dev").unwrap().unwrap(); + let visibility = graph_visibility_closure(&txn, &view).unwrap(); + let options = + atomic_core::output::alive::RetrieveOptions::new().with_graph_visibility(visibility); + let (full, _, had_fork) = atomic_core::output::repo::output_file_to_buffer_with_options( + &txn, + repo.change_store(), + position, + Default::default(), + options.clone(), + ) + .unwrap(); + assert_eq!(full, revision.as_bytes()); + for cap in [0, 1, full.len() - 1, full.len(), full.len() + 1] { + let bounded = atomic_core::output::repo::output_file_to_buffer_bounded( + &txn, + repo.change_store(), + position, + options.clone(), + cap, + ) + .unwrap(); + assert_eq!(bounded.had_fork_structure, had_fork); + assert!(bounded.buffer_high_water <= cap); + assert_eq!( + bounded.content.as_deref(), + (cap >= full.len()).then_some(full.as_slice()) + ); + match retrieve_content_bounded( + &txn, + repo.change_store(), + inode, + position, + options.clone(), + cap, + ) + .unwrap() + { + BoundedContent::Complete(bytes) => assert_eq!(bytes, full), + BoundedContent::Oversized { retained_bytes } => { + assert!(cap < full.len()); + assert!(retained_bytes <= cap); + } + } + } + } + repo.create_view_from("bounded-side", "dev").unwrap(); + repo.switch_view("bounded-side").unwrap(); + std::fs::write(temp.path().join("source.rs"), "first\nside\nthird\n").unwrap(); + record(&repo, "side insertion"); + repo.switch_view("dev").unwrap(); + std::fs::write(temp.path().join("source.rs"), "first\nmain\nthird\n").unwrap(); + record(&repo, "main insertion"); + repo.insert_from_view(CrossViewInsertOptions::new("bounded-side", "dev")) + .unwrap(); + let txn = repo.pristine().read_txn().unwrap(); + let inode = txn.get_inode("source.rs").unwrap().unwrap(); + let position = txn.inode_position(inode).unwrap().unwrap(); + let view = txn.get_view("dev").unwrap().unwrap(); + let options = atomic_core::output::alive::RetrieveOptions::new() + .with_graph_visibility(graph_visibility_closure(&txn, &view).unwrap()); + let (full, _, had_fork) = atomic_core::output::repo::output_file_to_buffer_with_options( + &txn, + repo.change_store(), + position, + Default::default(), + options.clone(), + ) + .unwrap(); + assert!(had_fork, "fixture must exercise semantic fork resolution"); + for cap in [0, full.len() - 1, full.len(), full.len() + 1] { + let bounded = atomic_core::output::repo::output_file_to_buffer_bounded( + &txn, + repo.change_store(), + position, + options.clone(), + cap, + ) + .unwrap(); + assert_eq!(bounded.had_fork_structure, had_fork); + assert_eq!( + bounded.content.as_deref(), + (cap >= full.len()).then_some(full.as_slice()) + ); + match retrieve_content_bounded( + &txn, + repo.change_store(), + inode, + position, + options.clone(), + cap, + ) + .unwrap() + { + BoundedContent::Complete(bytes) => assert_eq!(bytes, full), + BoundedContent::Oversized { retained_bytes } => { + assert!(cap < full.len()); + assert!(retained_bytes <= cap); + } + } + } +} + +#[test] +fn kg_content_window_and_nonfile_path_scoring_are_preserved() { + let (temp, repo) = create_temp_repo(); + for (path, lines) in [("src/a.rs", 1999), ("src/z.rs", 2), ("tests/a.rs", 10)] { + let full = temp.path().join(path); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(full, "windowmarker\n".repeat(lines)).unwrap(); + repo.add(path, TrackingOptions::default()).unwrap(); + } + record(&repo, "promotion window fixture"); + let project = repo.project_id().unwrap(); + let memory_id = iri::mint_memory(&project, "pathless"); + let path_id = iri::mint_goal(&project, "withpath"); + let mut txn = repo.pristine().write_txn().unwrap(); + txn.upsert_kg_node(&KgNode::new( + &memory_id, + Class::Memory, + "windowmarker", + "vault", + )) + .unwrap(); + txn.upsert_kg_node( + &KgNode::new(&path_id, Class::Goal, "windowmarker", "vault") + .with_type(Class::Memory) + .with_metadata(serde_json::json!({"path": "src/z.rs"})), + ) + .unwrap(); + txn.commit().unwrap(); + let nodes = repo + .vault_kg_search_on_view("dev", "windowmarker", 20, None) + .unwrap(); + assert_eq!( + nodes + .iter() + .find(|node| node.id == iri::mint_file(&project, "src/a.rs")) + .unwrap() + .metadata + .as_ref() + .unwrap()["content_matches"], + 1999 + ); + assert_eq!( + nodes + .iter() + .find(|node| node.id == iri::mint_file(&project, "src/z.rs")) + .unwrap() + .metadata + .as_ref() + .unwrap()["content_matches"], + 1 + ); + assert!(!nodes + .iter() + .any(|node| node.id == iri::mint_file(&project, "tests/a.rs"))); + let filtered = repo + .vault_kg_search_on_view_with_kind( + "dev", + "windowmarker", + 20, + None, + Some(Class::Memory.iri()), + ) + .unwrap(); + assert_eq!(filtered[0].id, path_id); + assert_eq!(filtered[0].metadata.as_ref().unwrap()["content_matches"], 1); + let (nodes, resources) = crate::query_diagnostics::measure(|| { + repo.vault_kg_search_on_view_with_kind( + "dev", + "windowmarker", + 20, + None, + Some(Class::View.iri()), + ) + .unwrap() + }); + assert!(nodes.is_empty()); + assert_eq!(resources.files_retrieved, 0); + // A genuinely vault-only match cannot receive a content score. + let mut txn = repo.pristine().write_txn().unwrap(); + txn.upsert_kg_node(&KgNode::new( + &memory_id, + Class::Memory, + "vaultonlymarker", + "vault", + )) + .unwrap(); + txn.commit().unwrap(); + let (nodes, resources) = crate::query_diagnostics::measure(|| { + repo.vault_kg_search_on_view_with_kind( + "dev", + "vaultonlymarker", + 1, + Some(1), + Some(Class::Memory.iri()), + ) + .unwrap() + }); + assert_eq!(nodes[0].id, memory_id); + assert_eq!(resources.files_retrieved, 0); +} + fn record(repo: &TestRepository, message: &str) -> Hash { let author = if message == "sibling_history_marker" { "HiddenSiblingAuthor" @@ -277,6 +654,16 @@ fn invalid_dependency_closure_fails_closed_for_every_query() { .unwrap_err(), repo.search_content_on_view("feature", "sibling", ContentSearchOptions::default()) .unwrap_err(), + repo.search_content_paths_on_view( + "feature", + "sibling", + ContentSearchOptions { + max_results: Some(0), + ..Default::default() + }, + &["missing/file.rs".into()], + ) + .unwrap_err(), repo.query_callers_on_view("feature", "src/lib.rs#target") .unwrap_err(), repo.vault_kg_search_on_view_with_kind( diff --git a/atomic-repository/src/repository/vault_triples.rs b/atomic-repository/src/repository/vault_triples.rs index 787d9ed2..2fbcce60 100644 --- a/atomic-repository/src/repository/vault_triples.rs +++ b/atomic-repository/src/repository/vault_triples.rs @@ -296,26 +296,20 @@ impl Repository { query: &str, limit: usize, pool: Option, - scope: Option<&super::query::QueryView>, + scope: Option<&super::query::QueryView<'_>>, kind: Option<&str>, ) -> Result, RepositoryError> { let kind = kind.filter(|kind| !kind.is_empty()); - let txn = self.pristine.read_txn()?; + let unscoped_txn; + let txn = if let Some(scope) = scope { + &scope.txn + } else { + unscoped_txn = self.pristine.read_txn()?; + &unscoped_txn + }; let mut rows = BTreeMap::new(); let mut content = HashMap::::new(); - if let Some(scope) = scope { - if let Ok(result) = scope.search_content( - query, - crate::content_search::ContentSearchOptions { - max_results: Some(2000), - ..Default::default() - }, - ) { - for result in result.matches { - *content.entry(result.path).or_default() += 1; - } - } - } else if crate::content_search::has_content_index(self.root()) { + if scope.is_none() && crate::content_search::has_content_index(self.root()) { if let Ok(result) = crate::content_search::search_content( self.root(), query, @@ -332,7 +326,7 @@ impl Repository { for (id, _) in txn.kg_fts_match_ids(query)? { if let Some(node) = txn.get_kg_node(&id)? { let node = if let Some(scope) = scope { - let Some(node) = scope.project_node(&txn, node)? else { + let Some(node) = scope.project_node(txn, node)? else { continue; }; node @@ -353,6 +347,30 @@ impl Repository { rows.insert(node.id.clone(), (node.clone(), hits)); } } + // File promotion always needs the original global 2,000-match + // window. Other types need it only when a surviving candidate has + // a source path that can receive content scoring. Do not suppress + // such scores on multi-typed/path-bearing non-File nodes. + let needs_content = kind.is_none_or(|kind| kind == Class::File.iri()) + || rows.values().any(|(node, _)| { + kind.is_none_or(|kind| node.types.iter().any(|ty| ty == kind)) + && node.path().is_some_and(|path| scope.contains_file(path)) + }); + if needs_content { + // KG keyword search historically ignores invalid regex patterns, + // but actual selected-source failures must still propagate. + if let Ok(plan) = scope.content_plan( + query, + crate::content_search::ContentSearchOptions { + max_results: Some(2000), + ..Default::default() + }, + ) { + for result in scope.search_content_plan(plan)?.matches { + *content.entry(result.path).or_default() += 1; + } + } + } } let project = self.project_id()?; for path in content.keys() { diff --git a/atomic-repository/tests/query_view_performance.rs b/atomic-repository/tests/query_view_performance.rs new file mode 100644 index 00000000..671b6307 --- /dev/null +++ b/atomic-repository/tests/query_view_performance.rs @@ -0,0 +1,116 @@ +//! Opt-in diagnostic, not a machine-dependent timing gate. +#![cfg(feature = "query-test-instrumentation")] + +use atomic_core::change::ChangeHeader; +use atomic_repository::{ContentSearchOptions, RecordOptions, Repository, TrackingOptions}; +use std::time::{Duration, Instant}; + +#[test] +#[ignore = "release diagnostic: 2000 recorded sources and five measured runs"] +fn recorded_query_resources() { + let dir = tempfile::tempdir().unwrap(); + let mut repo = Repository::init(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + std::fs::create_dir_all(dir.path().join("src")).unwrap(); + for index in 0..2000 { + let path = format!("src/file_{index:04}.rs"); + let content = format!("// needle {index}\n{}\n", " ".repeat(4080)); + std::fs::write(dir.path().join(&path), content).unwrap(); + repo.add(wc, &path, TrackingOptions::default()).unwrap(); + } + for (path, content) in [ + ("src/dense.rs", "needle\n".repeat(100_000)), + ("cutoff.txt", "x".repeat(10 * 1024 * 1024)), + ("oversized.txt", "x".repeat(10 * 1024 * 1024 + 1)), + ] { + std::fs::write(dir.path().join(path), content).unwrap(); + repo.add(wc, path, TrackingOptions::default()).unwrap(); + } + repo.record( + wc, + ChangeHeader::new("query benchmark fixture"), + RecordOptions::new() + .with_all(true) + .with_max_file_size(11 * 1024 * 1024) + .save_to_store(true) + .apply_after_record(true), + ) + .unwrap(); + let root = repo.desired_view_name(wc).unwrap(); + let mut parent = root; + for depth in 0..20 { + let child = format!("query-bench-{depth}"); + repo.create_draft_view(&child, &parent).unwrap(); + parent = child; + } + println!("fixture=2000 sources (~4KiB), 100000-line dense, cutoff/cutoff+1, 20 ancestors; setup excluded; warm ChangeStore/OS caches; no cache flush"); + for (label, paths) in [ + ("exact", vec!["src/file_1999.rs".into()]), + ("prefix", vec!["src/".into()]), + ("all", vec![]), + ] { + for limit in [0, 1, 50] { + let run = || { + repo.search_content_paths_on_view( + &parent, + "needle", + ContentSearchOptions { + max_results: Some(limit), + ..Default::default() + }, + &paths, + ) + .unwrap() + }; + run(); + let mut durations = Vec::::new(); + let mut last = None; + for _ in 0..5 { + let start = Instant::now(); + let (result, resources) = atomic_repository::query_diagnostics::measure(run); + durations.push(start.elapsed()); + last = Some((result.total_matches, resources)); + } + durations.sort(); + let (total, resources) = last.unwrap(); + println!( + "{label} limit={limit} total={total} min={:?} median={:?} max={:?} resources={}", + durations[0], + durations[2], + durations[4], + serde_json::to_string(&resources).unwrap() + ); + // Set QUERY_BENCH_BASELINE=1 only for the pre-fix diagnostic run. + if std::env::var_os("QUERY_BENCH_BASELINE").is_none() { + assert_eq!(total, if label == "exact" { 1 } else { 102_000 }); + assert_eq!( + resources.files_retrieved, + match label { + "exact" => 1, + "prefix" => 2001, + _ => 2003, + } + ); + assert!(resources.output_buffer_high_water <= 10 * 1024 * 1024 + 1); + assert!(resources.displayed_records_high_water <= limit); + } + } + } + let result = repo + .search_content_paths_on_view( + &parent, + "^x", + ContentSearchOptions { + max_results: Some(1), + ..Default::default() + }, + &["cutoff.txt".into(), "oversized.txt".into()], + ) + .unwrap(); + assert_eq!( + result.total_matches, 1, + "exact cutoff is searchable, cutoff+1 is not" + ); + assert_eq!(result.matches[0].path, "cutoff.txt"); + assert_eq!(result.matches[0].line_content.len(), 10 * 1024 * 1024); +} diff --git a/docs/query-view-performance.md b/docs/query-view-performance.md new file mode 100644 index 00000000..247f1859 --- /dev/null +++ b/docs/query-view-performance.md @@ -0,0 +1,111 @@ +# Recorded-query resource diagnostic + +Intent 11 optimizes recorded `query code` and KG content promotion. It does not +change the existing ambient ignore policy, graph visibility, or persistent indexes. + +## Reproduce + +Run serially, with no competing builds replacing the executable: + +```sh +rustc --version +git rev-parse HEAD +cargo test -p atomic-repository --features query-test-instrumentation --test query_view_performance --release --locked -- --ignored --test-threads=1 --nocapture +``` + +The ignored diagnostic constructs 2,000 approximately 4 KiB recorded sources, a +100,000-line dense source, exactly-10-MiB and cutoff+1 files, and a 20-level view +chain. Setup is outside the timings. Each exact-file, prefix, and all-files query +runs at display limits 0, 1, and 50, with one warmup and five measured repeats. +It reports minimum/median/maximum, exact totals, files retrieved, cumulative bytes +retained, consumer output-buffer high-water, and displayed-record high-water. +These are warm ChangeStore/OS-cache measurements, not cold-disk claims. No cache +flush is attempted. Deterministic resource assertions, not elapsed time, are gates. + +Instrumentation is thread-local and opt-in through `query-test-instrumentation` +(also enabled for repository unit tests). It measures synchronous calls on their +calling thread; it does not propagate into service worker threads. Default builds +do not expose the measurement API. No public query response or protobuf changes +are needed. + +## Cost and compatibility contract + +- Path-union and existing syntext glob/type/exclude filtering precede content I/O. +- View closure, path claims, and eligible inode attributes are validated before + request filters. Narrow searches still fail on invalid global metadata. +- Content-only corruption outside the selected paths is deliberately unread; + selected content errors propagate rather than falling back to disk or an index. +- Exact totals and directory counts require scanning every eligible file, even + when displaying zero results. Top-ten facet ties sort by directory path. +- The consumer retains at most one file output of 10 MiB+1, and no more than the + requested number of display records. Oversized content is not searched as a + truncated file. A matching line can itself reach the raw-byte cutoff; lossy + UTF-8 display can expand those bytes in retained result text. +- Graph/visibility metadata, ChangeStore caches, renderer span buffers, and + semantic-merge scratch are separate costs, not included in the consumer bound. + No claim is made that the whole request uses constant memory or O(limit) time. +- KG search preserves its ranked 2,000-match promotion window and per-file + `content_matches` metadata, rather than substituting exhaustive per-file counts. + Vault-only class searches do not read irrelevant source content. +- Caller parsing retains its full-source requirements; it is not made streaming + or bounded-memory by this optimization. No persistent cache is introduced. + +## Measurements + +Measured on an Apple M5 Max, 48 GiB RAM, macOS, Rust 1.98.0 +(`88d9e12ae`, 2026-08-18), with the checked-in Cargo.lock and release profile. +Both stages used Git base `e489ed4356c047ae7ec355a4a25ac0527c10ffcf` plus local +intent-11 work; these are **uncommitted source stages**, not two invented commit +hashes. The baseline was run before the selective projection/collector rewrite, +with test instrumentation and dormant bounded-rendering APIs present. The final +stage used the new metadata-only projection, capped retrieval and streaming +collector. Release stages ran serially, so one build could not replace another +stage's running executable. The final table uses a foreground rerun after other +background checks had completed, with no concurrent builds. These elapsed times +remain diagnostic, not a machine-independent performance promise. + +The final inspected query-source stamp is SHA-256 +`cec225e96cf73ca517dd03fec9d735ce76e2a6a156605fb7956b016f5bc1ea14`. +It hashes sorted path + NUL + file bytes + NUL for `Cargo.lock`, +`atomic-core/src/output/repo/{file,mod}.rs`, `atomic-repository/Cargo.toml`, +`atomic-repository/src/{content_search,query_diagnostics}.rs`, +`atomic-repository/src/repository/{content,query,vault_triples}.rs`, and +`atomic-repository/tests/query_view_performance.rs`. The stamp includes the final +documentation/derive cleanups and the shared comparator; these do not alter the +recorded-query execution measured by the final benchmark. + +The baseline command additionally set `QUERY_BENCH_BASELINE=1`, which disables +only the new resource assertions in this ignored test. That variable is not read +by production code. The failed initial fixture setup hit the record API's default +10-MiB limit; the benchmark explicitly raises that setup-only limit to 11 MiB to +record the deliberately oversized test file. It does not change search's cutoff. + +Times below are median **[minimum–maximum]**, in milliseconds, over five warm +runs after one warmup. Setup and compilation are excluded. + +| Scope | Display limit | Before (ms) | After (ms) | +|---|---:|---:|---:| +| Exact file | 0 | 183.45 [182.01–186.07] | 6.42 [6.32–6.49] | +| Exact file | 1 | 184.80 [182.95–191.16] | 6.46 [6.43–6.53] | +| Exact file | 50 | 185.40 [182.45–188.24] | 6.41 [6.40–6.43] | +| `src/` prefix | 0 | 212.64 [211.30–214.22] | 174.94 [173.79–175.59] | +| `src/` prefix | 1 | 213.68 [211.98–215.00] | 174.30 [172.68–178.97] | +| `src/` prefix | 50 | 210.87 [209.04–212.39] | 173.87 [173.40–174.34] | +| All files | 0 | 227.93 [213.20–272.88] | 177.74 [175.92–178.59] | +| All files | 1 | 219.28 [217.10–222.06] | 176.72 [175.38–178.10] | +| All files | 50 | 215.38 [211.24–221.32] | 177.46 [175.46–177.80] | + +All before/after totals agree: 1 for the exact query, 102,000 for prefix/all. +The exact query went from 2,003 retrievals / 29,862,411 cumulative retained bytes +to **1 retrieval / 4,096 bytes**. Prefix queries now retrieve 2,001 files / +8,890,890 cumulative bytes. All-files queries still retrieve 2,003 files, but +retain 19,376,650 cumulative bytes because oversized content is rejected before +fully collecting it. Cumulative bytes are not simultaneous resident memory. + +For prefix/all searches, displayed-record high-water fell from **102,000** to +**0 / 1 / 50** at those limits. The consumer-buffer high-water is now 4,096 bytes +for exact, 700,000 for prefix, and 10,485,760 for all-files queries. Those counters +exclude separately identified renderer/graph/ChangeStore costs; this is not an +RSS benchmark or a claim that the whole request is constant-memory. Machine load +and filesystem caches affect timings, so the diagnostic gates resource counts +and result equivalence, not these elapsed-time numbers. diff --git a/libatomic/tests/query_view_service_test.rs b/libatomic/tests/query_view_service_test.rs index 6483fbde..6eb6444c 100644 --- a/libatomic/tests/query_view_service_test.rs +++ b/libatomic/tests/query_view_service_test.rs @@ -626,3 +626,157 @@ async fn tonic_kind_filter_is_applied_before_pool_and_result_limits() { stop.send(()).unwrap(); server.await.unwrap(); } + +#[tokio::test] +async fn bounded_content_queries_preserve_exact_results_across_all_boundaries() { + use libatomic::daemon::{services_agent::KnowledgeImpl, state::DaemonState}; + use std::sync::Arc; + let (dir, local, local_repository) = fixture().await; + let repo = Repository::open(dir.path()).unwrap(); + let wc = repo.require_working_copy_id().unwrap(); + for (path, bytes) in [ + ("docs/a.md", b"needle\n".repeat(2500)), + ("src/z.rs", b"prefix needle needle\r\n\r\nneedle\n".to_vec()), + ("tests/a.rs", b"needle\n".repeat(3)), + ] { + let full = dir.path().join(path); + std::fs::create_dir_all(full.parent().unwrap()).unwrap(); + std::fs::write(full, bytes).unwrap(); + repo.add(wc, path, TrackingOptions::default()).unwrap(); + } + record(&repo, "dense and late source-tier matches"); + let before = std::fs::read(dir.path().join("src/lib.rs")).unwrap(); + let graph_before = serde_json::to_value(repo.workflow_snapshot().unwrap()).unwrap(); + let index_before = + std::fs::read(dir.path().join(".atomic/content-index/manifest.json")).unwrap(); + drop(repo); + let state = Arc::new(DaemonState::new()); + let repository = state + .register(dir.path().canonicalize().unwrap()) + .repository_ref(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (stop, stopped) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(pb::knowledge_service_server::KnowledgeServiceServer::new( + KnowledgeImpl { state }, + )) + .serve_with_incoming_shutdown( + tokio_stream::wrappers::TcpListenerStream::new(listener), + async { + let _ = stopped.await; + }, + ) + .await + .unwrap(); + }); + let mut client = + pb::knowledge_service_client::KnowledgeServiceClient::connect(format!("http://{address}")) + .await + .unwrap(); + for name in ["dev", "feature"] { + for limit in [0, 1, 3] { + for paths in [ + vec![], + vec!["src/".into(), "tests/".into()], + vec!["missing".into()], + ] { + let content = + pb::query_graph_request::Query::ContentSearch(pb::ContentSearchQuery { + query: "needle".into(), + paths: paths.clone(), + max_results: Some(limit), + ..Default::default() + }); + let repo = Repository::open_readonly(dir.path()).unwrap(); + let expected = repo + .search_content_paths_on_view( + name, + "needle", + atomic_repository::ContentSearchOptions { + max_results: Some(limit as usize), + ..Default::default() + }, + &paths, + ) + .unwrap(); + drop(repo); + let actual = query(&local, &local_repository, Some(view(name)), content.clone()) + .await + .unwrap(); + let over_wire = client + .query_graph(pb::QueryGraphRequest { + repository: Some(repository.clone()), + view: Some(view(name)), + limit: 1, + query: Some(content), + }) + .await + .unwrap() + .into_inner(); + assert_eq!(actual, over_wire); + let result = actual.content_result.unwrap(); + assert_eq!(result.total_matches, expected.total_matches as u64); + assert_eq!(result.matches.len(), expected.matches.len()); + assert_eq!( + result + .dir_facets + .iter() + .map(|facet| (facet.path.clone(), facet.count as usize)) + .collect::>(), + expected.dir_facets + ); + for (actual, expected) in result.matches.iter().zip(expected.matches) { + assert_eq!(actual.path, expected.path); + assert_eq!(actual.line_number, expected.line_number); + assert_eq!(actual.byte_offset, expected.byte_offset); + assert_eq!(actual.line_content, expected.line_content); + assert_eq!(actual.submatch_start, expected.submatch_start as u64); + assert_eq!(actual.submatch_end, expected.submatch_end as u64); + } + if paths.is_empty() && limit > 0 { + assert_eq!( + result.matches[0].path, "src/z.rs", + "late source must outrank earlier docs" + ); + assert_eq!(result.total_matches, 2505); + } + } + } + } + let bad = pb::query_graph_request::Query::ContentSearch(pb::ContentSearchQuery { + query: "[".into(), + paths: vec!["missing".into()], + max_results: Some(0), + ..Default::default() + }); + assert!(query(&local, &local_repository, None, bad.clone()) + .await + .is_err()); + assert!(client + .query_graph(pb::QueryGraphRequest { + repository: Some(repository), + query: Some(bad), + ..Default::default() + }) + .await + .is_err()); + assert_eq!( + std::fs::read(dir.path().join("src/lib.rs")).unwrap(), + before + ); + assert_eq!( + std::fs::read(dir.path().join(".atomic/content-index/manifest.json")).unwrap(), + index_before + ); + let repo = Repository::open_readonly(dir.path()).unwrap(); + assert_eq!( + serde_json::to_value(repo.workflow_snapshot().unwrap()).unwrap(), + graph_before + ); + assert_eq!(repo.desired_view_name(wc).unwrap(), "dev"); + drop(client); + stop.send(()).unwrap(); + server.await.unwrap(); +}