diff --git a/code-mower-package-manifest.json b/code-mower-package-manifest.json index bd746e35..1ec18edf 100644 --- a/code-mower-package-manifest.json +++ b/code-mower-package-manifest.json @@ -107,6 +107,11 @@ "source": "docs/context-graph-lifecycle.md", "target": "docs/context-graph-lifecycle.md" }, + { + "kind": "doc", + "source": "docs/context-graph-queries.md", + "target": "docs/context-graph-queries.md" + }, { "kind": "doc", "source": "docs/context-packet-schema.md", @@ -727,11 +732,21 @@ "source": "src/code_mower/context_graph_command.py", "target": "src/code_mower/context_graph_command.py" }, + { + "kind": "core", + "source": "src/code_mower/context_graph_connection.py", + "target": "src/code_mower/context_graph_connection.py" + }, { "kind": "core", "source": "src/code_mower/context_graph_lifecycle.py", "target": "src/code_mower/context_graph_lifecycle.py" }, + { + "kind": "core", + "source": "src/code_mower/context_graph_query.py", + "target": "src/code_mower/context_graph_query.py" + }, { "kind": "core", "source": "src/code_mower/context_guided.py", diff --git a/docs/context-delivery.md b/docs/context-delivery.md index bf3dfe28..a01416c3 100644 --- a/docs/context-delivery.md +++ b/docs/context-delivery.md @@ -161,7 +161,41 @@ that exact revision from the trusted current comment. If a write returns an uncertain result and the revision is not current, status pauses the workflow; after independently checking the PR, `attach --retry-uncertain` republishes the same revision rather than creating another one. No uncertain or unpublished -binding can deliver evidence. +binding can deliver evidence. A saved uncertain or pending intent is never +cleared just because a later attach was asked for: it may already be the +GitHub-accepted state a lost response only looked like it missed. + +Before attempting a fresh reservation, the session first saves a durable +`reserving` marker -- proof that no GitHub write has happened yet, because +publication only ever follows a reservation that has itself already become +durable. If that reservation then fails within the same `attach` call -- for +example because the actual consuming checkout has moved past the evidence it +was prepared from, or the local graph was rebuilt in the meantime -- the same +call rolls the session back to its prior prepared, unattached state inline, +keeping the failure reason visible in status. If the process instead stops +before that same-call rollback, or before the reservation's own durable +transition out of `reserving` completes -- a crash, or a storage fault like +the one that also surfaces as a `ContextError` from a saved-state transition +that did not complete -- the saved intent is left as `reserving` instead. The +next `attach` recognizes a saved `reserving` marker, finishes abandoning it +and clearing the session back to its prior prepared, unattached state, and +asks the caller to rerun attach rather than completing a fresh attempt in +that same call, since resuming immediately could race a concurrent recovery. +Only once that cleanup has actually finished is the session safe to +explicitly rebuild, rerun `prepare --refresh`, or attach again; refresh must +not be used while a `reserving` cleanup is still outstanding, and a storage or +other fault surfaced while attach itself was trying to advance a saved intent +does not promise that this cleanup has already happened -- rerunning attach is +what finishes it. + +A saved `pending` or `uncertain` intent is different: its reservation is +already durable, and GitHub publication may have begun or even completed +before a lost response left the local state unconfirmed, so it is never +blanket-cleared the way a `reserving` marker is. Once the trusted current pull +request head has genuinely moved past such a saved intent, that old identity +is retired before any new evidence is authorized, so a checkout that has since +moved on cannot block its own recovery. This retirement, like the `reserving` +cleanup above, is safe to retry after an interruption partway through. The lower-level expert form remains available for scripts that intentionally manage request files and revisions: diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index 21240dae..0fee5551 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -44,9 +44,10 @@ provenance at all. If Code Mower does not bind the revision, nothing does. repository it was never authorized to read. Committed private state is skipped for a third reason, at any depth and case-folded: the roots are `context_graph`'s excluded roots themselves — `.git`, `.graph`, `.graphify`, - `.code-mower` — bound rather than copied, so the set that refuses a citation - into private state is the same set that keeps those bytes away from the - indexer. A tracked `.graphify/` or `.graph/` is somebody's old index, and + `graphify-out`, `.code-mower` — bound rather than copied, so the set that + refuses a citation into private state is the same set that keeps those bytes + away from the indexer. A tracked `.graphify/`, `.graph/` or `graphify-out/` + is somebody's old index, and materializing it would let the provider resume from a cache built over content this build never saw, and let the adapter collect tracked repository bytes as if the provider had just produced them. A tracked `.code-mower/` @@ -180,6 +181,69 @@ being read out of a file makes a path no narrower than guessing it would — and an environment that records no base that still exists is refused with an instruction rather than built against whatever runtime is lying around. +A prefix is still not the whole runtime on a host whose interpreter came from a +package manager. CPython's `_ssl` extension lives inside the interpreter's +prefix and is *linked against* an OpenSSL that does not — under Homebrew, +`/opt/homebrew/opt/openssl@3/lib/libssl.3.dylib` and the `libcrypto` beside it — +and Graphify imports `ssl` during start-up even for a code-only extraction. With +only the prefixes exposed those libraries are simply absent, so the provider +aborted inside the loader before it scanned anything. That is a missing runtime +dependency; it is not an argument for giving the child a network or a wider +filesystem, and neither was granted. + +So the dependency is **derived, never named**. Every Mach-O image inside the +exposure is read for the libraries its own load commands ask `dyld` to find, +each one not already covered is resolved and added, and newly added libraries +are read in turn, so `libssl` needing `libcrypto` is reached without either +being written down. What is added is the **library file**, never the directory +holding it: exposing `/opt/homebrew/opt/openssl@3/lib` exposes a package +manager's prefix, and its `etc` and `var` with it. `@rpath`, `@loader_path` and +`@executable_path` names are resolved by `dyld` against the image itself and are +not paths this adds. Every derived path goes through the same broad-exposure +refusals as any other exposure — the filesystem root, the operator's home, the +checkout, an ancestor of either — and is additionally refused unless it is a +regular file within a size bound whose ancestry only this account or root may +write, because a library the provider maps executable inside the boundary is +code. It must also **say it is a shared library in its own header**: the name +came out of a load command in somebody else's image, and ownership and size say +who wrote a file and how big it is, not what it is — so without reading the +container, a provider that writes its own linker input picks which of the +operator's files this boundary exposes, and each one passes every check an +operator-owned file passes. The candidate is read as a Mach-O and its +`filetype` must be `MH_DYLIB` or the `MH_DYLIB_STUB` a stripped SDK ships in its +place. Not an executable, an object file or an `MH_BUNDLE`, which is reached +through `dlopen` rather than through the loader these commands drive. A +universal archive is held to that on **every** slice — the child is the +provider's interpreter, whose architecture is not necessarily this one, so the +slice loaded inside the boundary is not the slice a single check would pick — +and a file that is not a readable Mach-O at all is refused as such. + +A magic and a `filetype` field are not that reading. Those four bytes and that +one integer are the cheapest thing in the file to reproduce over arbitrary +operator-owned bytes, so the **structure behind them** is what is checked. The +universal header is decoded in the byte order its own magic declares — +`FAT_MAGIC` big-endian, `FAT_CIGAM` little — because reading the swapped +spelling as big-endian turns a count of two into 33 million and every slice +offset into a number with no relation to the file. The architecture table is +bounded and must fit in the file; each slice must begin after the table +describing it, end within the file, and not overlap another slice, because +overlapping slices make "which image is this" ambiguous. Within each admitted +slice the thin header must be complete and its declared load-command region +must fit inside **that slice** rather than merely inside the file, so one slice +cannot reach into the next one's bytes to satisfy its header. The commands then +have to walk: each `cmdsize` at least a command header, a multiple of the +image's pointer width, and within the region — and the chain must consume the +region exactly, since `sizeofcmds` is the size of *all* the commands and a +chain that stops short leaves unexamined bytes where only commands belong. A +64-bit universal header (`FAT_MAGIC_64`) has wider records and is refused as +unrecognized rather than guessed at. Anything malformed is refused whole rather +than read as far as it parses. A +referenced path that this host does not have installed is skipped: if it +turns out to have been required, the loader fails the build naming the library +it could not find. The derivation reads Mach-O images, so **Linux is unchanged** +— an ELF runtime's libraries are already under the `/lib` and `/usr/lib` +directories the read-only runtime names. + The prefix a particular build ends up with is probed before that build runs, not just the host's mechanism at startup: the readable set of a real build is the provider's install rather than the interpreter paths the host probe uses, @@ -294,6 +358,7 @@ Every published generation carries, in `manifest.json`: | `graph_digest`, `graph_bytes` | Detects a truncated or tampered artifact on every read. | | `completeness` | `complete` or `partial`, from the provider's own admission. | | `indexed_files` | What the provider claims it processed, bounded by the census. | +| `unsupported_inputs` | How many inputs the provider classified as code and then deterministically could not extract — no wired extractor, or an extractor that declined by design. Read bytes, no contribution. Kept separate from `skipped_paths`, which counts what this build declined to materialize at all; the two answer different questions and their sum answers neither. Optional on read so a generation published before the field existed still loads as `0`; always written. | `shareable_summary()` is the metadata-only view: revisions, digests, counts and states. It carries no indexed content, no provider output, and no local path. @@ -301,13 +366,26 @@ states. It carries no indexed content, no provider output, and no local path. ## How the provider is actually invoked `subprocess_indexer()` builds the argv for the interface the adopt decision -evaluated, not a conventional-looking one: `extract` plus the pinned options, -run with its working directory set to the materialized copy. The evaluated -release takes no `--source`/`--output` pair — `extract` reads the directory it -is run in and writes its state beside those sources, which the clean-room run in +evaluated, not a conventional-looking one: `extract`, the scan target, then the +pinned options, run with its working directory set to the materialized copy. The +evaluated release takes no `--source`/`--output` pair — `extract` writes its +state beside the sources it was pointed at, which the clean-room run in [the evaluation](graphify-evaluation.md) recorded as `extract --code-only --no-cluster --max-workers 4`. +**The scan target is required, and it is positional.** This module used to pass +the options alone, on the reading that a subcommand which writes beside its +sources must also discover them from the working directory. The pinned CLI does +not: it takes the target as the first positional after the subcommand, decides +it has one only when that argument does not begin with `-`, and exits 1 with +`must specify a path to scan or a --postgres DSN` when it does not. Every real +build therefore failed before extraction, and failed as the adapter's generic +non-zero refusal rather than as anything naming the omission. The target passed +is `.`: the child's working directory is already the materialized copy, so the +relative spelling names exactly that tree and names nothing about where it sits +on this host. It goes **between** the subcommand and the options, because the +CLI reads `sys.argv[2]` and nothing later. + **`--code-only` and `--no-cluster` are always passed, whatever the pin says.** They are conditions of the adopt decision, not preferences: a pin that named no options at all would otherwise have launched the provider into clustering and @@ -319,8 +397,12 @@ valued form such as `--code-only=false` — is refused rather than quietly overridden by argument order. So the adapter collects an artifact afterwards rather than naming one up front. -The state directory the provider wrote (`.graphify` or `.graph`, both already on -the excluded-roots list) is packed into a single reproducible archive: names +The output root the provider wrote — exactly `graphify-out`, the literal default +of the pin's own `GRAPHIFY_OUT`, resolved beneath the materialized copy and +never from a caller path or that environment override, and required to be a real +non-symlinked directory holding `graph.json` as a regular file, with a second +provider root beside it refused rather than guessed between — is packed into a +single reproducible archive: names sorted, timestamps and ownership fixed, modes normalized, symlinks dropped. Two builds of one commit have to produce identical bytes, because the manifest binds a digest of them. That state lands inside the throwaway materialized copy, never @@ -340,26 +422,102 @@ build from this module there is none; the refusal is the second check, because everything after the run treats whatever is in that directory as output this run produced. -**Completeness is read from the provider's report, never from its exit status, -and only an affirmative claim counts.** `complete` requires a report shaped the -way the adapter understands one: a claim that the run finished (`complete`, -`completed`, `finished`, or a recognized `status`), a count of what was -indexed, and no counter admitting requeued, pending, or failed work. Everything -else is `partial` — a report that denies completion, one in an unrecognized -schema, an empty object, an unreadable one, one larger than a manifest, and no -report at all. Absent evidence is not evidence of a complete build, and -`partial` is the state `graph_status` refuses by default, so the failure is one -an operator can see and act on. This is the direct consequence of the requeue -defect the evaluation recorded — a repeat that exits zero in 1.63 seconds -having requeued 54 entries has not built a complete graph. - -The report is provider output of unknown size, so it is read to one byte past +**Completeness is coverage, read from the provider's own manifest and never +from its exit status.** The pinned `save_manifest` writes a flat map of +repository-relative POSIX path to `{mtime, seen, ast_hash, semantic_hash}`. It +carries no completion flag and no indexed count, so there is no affirmative +claim to accept: the question is whether every input this pin would dispatch +came back with a hash proving the provider read its bytes. + +The denominator is the immutable materialized census narrowed to the pin's own +`detect.CODE_EXTENSIONS`. Inputs outside that set are deterministically not code +to this pin and are skipped rather than counted missing — holding a correct run +to every tracked Markdown file would make it permanently `partial`. + +A file counts as processed only when its row carries a well-formed 32-character +`ast_hash` **and that hash is the digest of the bytes this build actually handed +the provider**. The pin's `_md5_file` streams a file and returns the MD5 hex +digest of its contents, so the adapter takes the same digest of every eligible +input from the materialized copy *before* the launch — the only moment that tree +is still exactly what the provider was given — and compares. A well-formed +digest on its own only says a hash-shaped string is present; the comparison is +what says it is a hash of this input, and without it a manifest carried over +from another tree, another revision, or a resumed cache would read as proof of +work on bytes the provider never saw. + +The denominator is the census classified the way the pinned +`detect.classify_file` classifies it, **in its order** — not by extension +membership. A package manifest is routed by filename first (`apm.yml`, +`apm.yaml`, `pyproject.toml`, `Cargo.toml`, `go.mod`, `pom.xml`), an +extensionless file by its shebang second, and only then is the extension table +reached. Suffix membership alone misses both: `.yml`, `.toml`, `.mod` and +`.xml` are not code extensions, so a `pyproject.toml` the provider failed on +could drop out of the coverage question entirely while an unrelated `.py` file +let the run claim it was complete. + +Everything else is `partial`: a row the manifest never wrote, a blank or +malformed hash, a hash that disagrees with those bytes, an input the copy could +not be re-read for, a record this adapter cannot classify, an input it cannot +classify against the provider's own dispatch, an unreadable or oversized +manifest, no manifest at all, and a build with no census. + +### What a stamped row is evidence of + +From the pin's own post-extraction writer rule. After a run the CLI clears +(`clear_ast`) exactly the rows in `_failed_sources`, and stamps every other +dispatched input. A result is a failed source when it carries an `error`, or +when its extractor produced zero nodes. It is **not** when `_get_extractor` +returned `None` — the file short-circuits to an empty result with neither +marker — and **not** when the extractor declined by design, as the JSON +extractor does for data JSON and for a non-object root. + +So a stamped, matching row proves the provider read those exact bytes and did +not fail on them. It does not prove nodes. The deterministically unsupported +dispatch is therefore counted and reported on its own line +(`unsupported_inputs`) rather than folded into `indexed_files`, and zero nodes +for a stamped file is a complete *read* of that file and nothing more. + +### Where the boundary runs + +`classify_file` is the eligibility oracle. What it deterministically calls +not-code — every suffix outside its registry included — is not in the code +denominator, does not make a run partial, and is not counted anywhere; there is +no all-non-code skipped tally. `unsupported_inputs` counts the inputs it *does* +call code that then reach a dispatch with no extractor: the static table +difference, a code shebang with no dispatch entry, and a `.m` whose bytes carry +no Objective-C directive. That last one is decided from the bytes, not a table +— `.m` is Objective-C or MATLAB/Octave, the suffix map routes it to the +Objective-C extractor, and the pin returns no extractor for a `.m` without an +Objective-C directive rather than force-parsing MATLAB into garbage. The row is +stamped regardless, so counting it as an indexed file would report work that +did not happen. + +Everything else stays partial: an eligible code input that failed, one whose +postcondition is unknown, one whose row disagrees with the bytes it was given, +one this build could not classify against the pin's own dispatch, and a +zero-node result that cannot be told apart from a failure. + +Blank rows are the cases the pin's rule makes blank: an extractor error or an +anomalous zero-node extract. They stay `partial` here. The evaluation's +clean-room repeat that exited zero in 1.63 seconds requeued 54 entries; the +retained evidence for that run carries *stamped* rows, so requeueing there is +not observable as a blank row, and nothing in this module claims it is. That +requeue behaviour remains an observed limitation of the provider's incremental +gate rather than a shape this adapter reports on. + +Nothing upgrades a run. The exit status, a non-empty graph, and the raw +extraction's `extracted_sources` all describe what was *dispatched*, failures +included. An unstamped file is `partial` however large the graph is. `partial` +is the state `graph_status` refuses by default, so the failure is one an +operator can see and act on. + +The manifest is provider output of unknown size, so it is read to one byte past the manifest bound and refused if it is longer, rather than loaded whole and measured afterwards. A bound checked on bytes already in memory bounds nothing. The provider's own stdout and stderr are the other unbounded output, and they are discarded at the kernel: `stdin`, `stdout` and `stderr` are all -`DEVNULL`. Nothing reads them — completeness comes from the report, not from +`DEVNULL`. Nothing reads them — completeness comes from the manifest, not from what the run printed — so buffering them would only accumulate whatever a talkative indexer chose to log, for up to the timeout, under neither the tracked-content budget nor the artifact one. Inheriting them is not the diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md new file mode 100644 index 00000000..a8f2b801 --- /dev/null +++ b/docs/context-graph-queries.md @@ -0,0 +1,460 @@ +# Local repository graph: bounded queries and context packets + +How Code Mower asks a published local graph a question, and how the answer +becomes evidence a recipient may read. Recorded for +[issue #914](https://github.com/codemower-ai/code-mower/issues/914) under epic +[#902](https://github.com/codemower-ai/code-mower/issues/902). + +This is the third and last piece of the optional local-graph path: + +- [`context_graph_lifecycle.py`](context-graph-lifecycle.md) (#913) builds and + publishes an immutable generation bound to one commit and tree. +- `context_graph_query.py` (this document) reads the pinned schema out of that + generation, answers four bounded questions, and emits a packet. +- [`context_graph.py`](graphify-evaluation.md) (#876) scores a delivered + packet's citations and freshness. + +Nothing here installs, downloads, or runs a provider, and nothing here is on a +default path. The only subprocess is Git, reading blobs of the commit the +generation is already bound to. + +## Reading the pinned provider's own output + +A generation's artifact is the provider's own state, packed reproducibly. This +adapter reads one member of it, `graph.json`. There is no Code Mower graph +schema and no normalization pass between the build and the query: the lifecycle +archives what the provider wrote, so this is what gets read. + +### The supported document: the raw `--no-cluster` extraction + +`context_graph_lifecycle` runs the pinned release as +`extract --code-only --no-cluster` and accepts no other options +(`_REQUIRED_EXTRACT_OPTIONS`). At the pinned commit that branch of +`graphify/cli.py` dumps the merged extractor result straight to `graph.json` +through `write_json_atomic`. It never builds a NetworkX graph and never calls +`export.py::to_json`, so the document carries **no** `directed`, `multigraph`, +`graph`, `links` or `built_at_commit` key: + +```json +{ + "nodes": [ + {"id": "n-config", "label": "parse_config", "file_type": "code", + "source_file": "example_pkg/config.py", "source_location": "L12", + "type": "function", "metadata": {"namespace": "example_pkg"}} + ], + "edges": [ + {"source": "n-load", "target": "n-config", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "example_pkg/loader.py", + "source_location": "L41", "weight": 1.0} + ], + "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + "extracted_sources": ["example_pkg/config.py", "example_pkg/loader.py"] +} +``` + +Direction here is the extractor's own claim. `engine.py::add_edge` writes the +call site's `source` and `target`, and the raw dump carries that record through +unchanged, so reading the edge record reads the provider's semantics. Permuting +the node list cannot reverse a relationship, because nothing about the endpoint +order is derived from node order. + +An edge's own `source_file` and `source_location` are that call site — the file +and, where the extractor recorded one, the line the relationship was written +at. This adapter retains both, holds them to the same citation rules as a +node's location, and carries a resolvable one into the packet, so two call +sites of one relationship are two distinguishable pieces of evidence rather +than one relationship with the second one's location discarded. + +**Upstream limitation.** The pinned release's `--no-cluster` branch itself +calls `build.dedupe_edges`, keyed by `(source, target, relation)`, before +`graph.json` is written, so two call sites the extractor found while walking +the source may already be down to one record by the time this module reads the +artifact. What this adapter guarantees is narrower and still real: every edge +record the artifact actually carries — including two that differ only by call +site, which the supported directed node-link input carries the same way — is +preserved as a distinct relationship rather than collapsed a second time on the +way to a packet. It does not establish exhaustive live-provider call-site +coverage upstream of that dedupe step. + +### The other document: a NetworkX node-link export + +A generation built through the clustered path instead carries +`export.py::to_json`'s output — `networkx.json_graph.node_link_data`, which +always writes `directed`, `multigraph`, `graph` and a `links` list (NetworkX +later renamed that key to `edges`; the pinned validator accepts either). That +document is read too, but it is held to one extra requirement the raw format +does not need and does not get: + +> **`directed` must be `true`.** The clustered build stores an `nx.Graph` by +> default, and undirected storage canonicalizes endpoint order. `to_json` tries +> to repair this — the build stashes the true endpoints in `_src`/`_tgt` and the +> exporter restores them — but the written link carries no record that the +> repair happened, so a reader cannot distinguish a restored edge from one +> ordered by node iteration, and an undirected build additionally collapses a +> pair related in both directions onto whichever it saw first. Every answer this +> module produces is an oriented claim, so an undirected node-link export is +> refused rather than answered from. + +The two are told apart structurally, by the presence of a node-link marker key, +never by guessing from the name of the edge list — otherwise a newer NetworkX +export that names its links `edges` would be misread as a raw extraction and +skip the direction requirement it needs. + +`built_at_commit` is likewise a node-link-only stamp. Where it is present it +must equal the commit the generation is bound to; the raw path writes none, and +there the binding rests on the lifecycle's own commit binding and on the census +check every citation goes through. + +What is validated is the *provider's* contract, not one of ours, and it is the +same for both documents: + +- The required node and edge fields of `graphify/validate.py` — `id`, `label`, + `file_type`, `source_file` on a node; `source`, `target`, `relation`, + `confidence`, `source_file` on an edge. A record missing one would not have + passed the provider's own validator, so it is a refusal here. +- Its vocabularies. `file_type` must be one of the six it defines, and + `confidence` must be uppercase `EXTRACTED`/`INFERRED`/`AMBIGUOUS`. Lowercase + is the *packet* vocabulary, and a graph using it was not written by the + pinned provider. +- Its locations, on a node's `source_location` and an edge's own. Each is + `L` or empty; anything else — including a non-ASCII digit shape such as + `L²`, which reads as a digit but is not one `int()` can parse — is a location + this module could not check against the bound commit, so it refuses rather + than traversing past it or letting the shape escape as an unhandled error. + One line per node or call site, never a span: neither document records an + extent, and claiming one would be this adapter inventing it. A citable path — + a node's `source_file` or an edge's own — is held to the same repository- + scope rules either way: absolute, parent-traversing, and indexer-private + paths are refused, and a record whose path fails that check is not + traversable at all. + +Three things are deliberately *not* refusals, because a real generation carries +them and rejecting them would reject every ordinary one: + +- **Extra annotations.** The extractor adds `weight`, `context`, `type` and a + free-form `metadata` dict from an LLM extraction; a node-link export adds + `community`, `community_name` and `norm_label` to nodes and + `confidence_score` to links. None of them changes a traversal, so none is + read. Everything this module *does* read is read by name and bounded. +- **Relations outside the mapped set.** The provider's validator does not + constrain `relation` at all. Mapped relations (`calls`, `imports`, `defines`, + `contains`, `references`, `inherits`, `implements`, `tests`) decide which + traversals an edge participates in; anything else is grouped as `related`, + reachable only from the `symbol` neighbourhood. Either way the sentence in + the packet states the provider's own word, so an `implements` edge reads as + "implements" and a `supersedes` edge reads as "supersedes". +- **Sourceless stubs and non-code corpora.** The extractor emits nodes with an + empty `source_file` for cross-file references it could not resolve; those + stay traversable and are never cited. Nodes whose `file_type` is not `code` + are dropped, and edges onto a dropped node are pruned — which is the pinned + exporter's own treatment in `prune_dangling_edges`. No count of the dropped + records is kept or reported: what a packet says about its own incompleteness + is the traversal's truncation and omission fields, not a tally of corpora + this module never queries. + +Node kinds — `file`, `symbol`, `test` — are **derived**, not read: a Graphify +node declares its corpus and, rarely, a `type`, but never whether it is a file, +a definition, or a test. A file node is the one the extractor emits per file, +whose label is that file's base name; a test is a code node whose path sits in +this repository's test layout; everything else is a symbol. That derivation is +the one place this adapter infers something the provider did not state, and it +is named in `_node_kind` for that reason. + +Reading the member directly, rather than through provider query tooling, is what +makes the traversal reproducible and the bounds ours. It also means a recipient +never needs the provider: the graph is read once, here, by the operator who +built it. + +The member is read as a stream and bounded as it is read. The artifact as a +whole is already inside the lifecycle's budget and its digest was verified by +`graph_status` before this module opened it; one member of it is separately +bounded because a declared size is something this process would otherwise +allocate before looking at it. + +## The four questions + +Free-form traversal is not offered. A traversal whose shape comes from the +question text cannot be bounded or reproduced, and the adoption record's second +product constraint is that default traversals in the evaluated provider returned +700–900 nodes and truncated silently. + +| Question | Direction | Relationships | Default depth | +| --- | --- | --- | --- | +| `impact` | against the edges | `calls`, `imports`, `references`, `tests` | 2 | +| `dependency` | along the edges | `calls`, `imports`, `references` | 2 | +| `symbol` | both | all | 1 | +| `related_tests` | against the edges, answering with test nodes only | `tests`, `calls`, `references` | 2 | + +Each traversal is symbol-first: a target resolves to the symbols carrying that +name, and only a target that names no symbol at all is read as a path. Each is +breadth-first over adjacency sorted by the full relationship identity described +below — source, target, the provider's relation word, its normalized kind and +confidence, and the call site's path and line — so one generation and one +question produce one answer, every time; permuting the provider's own node or +edge order changes nothing about it; and a budget cut removes the furthest +relationships rather than arbitrary ones. + +A target resolves in three ordered tiers, and a later tier is consulted only +when every earlier one is empty: + +1. the provider's label, exactly as written; +2. the provider's canonical callable label with only its syntactic trailing + argument list removed; +3. the path, exactly as written. + +Tier 2 exists because the pinned extractor labels a callable with its argument +list: a real 0.9.58 graph of this repository names the function +`parse_graph_citation` as `parse_graph_citation()`. Without it, the natural bare +spelling of a function resolves to nothing and every question about it answers +"unresolved". It is an equality test against a label with one suffix stripped — +not a prefix, substring, or edit-distance match. `parse_graph` does not reach +`parse_graph_citation()`; an attribute `packet` and a function `packet()` stay +different definitions, and the exact label wins; and two overloads that differ +only in their argument lists both resolve, which is real ambiguity and is +reported as `unresolved_entities` rather than settled by picking one. + +Reaching the node budget sets `truncated` and raises `provider_has_more`. It +never silently shortens the answer. A target name carried by more definitions +than the seed bound allows does the same: seeds the bound drops take their whole +reachable neighbourhood out of the answer, so that is truncation too, not a +complete result over the seeds that happened to sort first. + +The depth limit is the third. A walk that stops at its requested depth on a node +that still has eligible relationships behind it has left evidence out, and +saying so is the whole point of the limit being explicit: for `a -> b -> c -> d` +a default `dependency` question about `a` ends at `c`, and a reader told that +answer is complete would conclude `c` depends on nothing. So the boundary sets +`truncated` and raises `provider_has_more`, exactly as the node budget does. + +It is not raised for reaching the boundary as such. Eligibility is measured the +way the walk measures it — this question's direction and relationship filter, +against relationships not already reported — so a chain that genuinely ends at +the boundary stays `complete`, and so does a cycle or a `symbol` neighbourhood +whose boundary edges were already stated from their other side. A parallel edge +the provider worded differently is a different relationship and does count. + +Every reported relationship is the one edge the walk crossed, between that +edge's own two endpoints. A second-hop result names the intermediate node and +cites it — `render calls load (inferred, hop 2, reached from parse_config, …)` — +rather than asserting a direct relationship between the seed and the node two +hops away, which the graph does not carry. + +What bounds the walk and what bounds the answer are two different things. A node +is stepped through once, which is what keeps a traversal linear and terminating. +A *relationship* is reported once per distinct provider edge record — its two +endpoints, its own wording, its normalized confidence, and its own call site +(path and, where the provider recorded one, line) together — including when +both endpoints have already been seen. So if `a` calls `b` and `b` calls `a`, +both directions are reported; relationships among the definitions a path target +selects as seeds are reported rather than dropped for having no unseen +endpoint; a walk that reconverges keeps both edges into the node it reached +twice; a self-loop reached from both sides of a `symbol` neighbourhood, or an +edge the provider recorded twice at the same call site, is one relationship; +and the same relationship recorded at two different call sites — two lines, or +the same line in two different files — is two, because the call site is part +of the identity rather than a detail dropped on the way to reporting it. +Nothing about the bound changes: a relationship that does not fit the node +budget still sets `truncated` and raises `provider_has_more`, and the depth +limit still applies and still reports what it stopped, including when what it +stopped on differs from an already-reported relationship only by call site. + +## Citations are validated against the bound commit + +Not against the working tree, which is the point. The generation binds one +commit; the checkout it was built from has since been edited, rebased, or left +dirty, and a line claim confirmed against an edited file is a claim confirmed +against a revision nobody asked about. + +Every cited path must appear in that commit's tracked census — so an untracked, +ignored, or since-deleted file is never cited — and every line claim is +confirmed against the blob the census names, read through `git cat-file` and +stopped at the claimed line. Blob line counts are memoized, so a packet that +cites one file many times reads it once. + +A location that cannot be confirmed is dropped rather than downgraded: the +packet's whole claim is that its citations point at the immutable tree, and +evidence that cannot be pointed at is not weaker evidence, it is none. The +drop is reported as `provider_warning`, and a relationship left with no citation +at all is reported as `document_limit`. + +The scope rules are `context_graph`'s, applied twice: at parse time, so a node +or an edge's own call site that could never be cited is not traversable either, +and again at citation time. + +## A dropped relationship is two different facts + +The provider's node list can be missing an edge's endpoint for two reasons, and +they are not reported the same way. + +The endpoint was **declared as a corpus this module does not query** — a +document, paper, image, rationale or concept. That is a scope stated in these +rules, the relationship is out of it, and the edge is pruned silently. No count +of those is kept: what a packet says about its own incompleteness is its +truncation and omission fields, not a tally of corpora never queried. + +The endpoint was **never declared at all**. The provider stated a relationship +and then described one of its ends nowhere, so this is evidence its own document +does not carry, not a scope this module chose. The surviving endpoint is +recorded, and a traversal that seeds or reaches that node reports +`provider_partial` and delivers a `partial` packet. It is not `truncated`: no +budget or depth limit cut it. Only the nodes the walk actually touches count — +a hole elsewhere in the repository is not a hole in this answer, and marking +every query partial for it would make the flag say nothing. + +## What the packet carries + +An ordinary `code_mower.contextPacket.v1` repository-kind packet — the same +shape every other provider delivers, so it travels the existing delivery path +with no new contract: + +| Field | Bound to | +| --- | --- | +| `source_revision` | the generation's commit, so a consumer asking about another revision resolves `stale` at delivery | +| `source_built_at` | the manifest's build time | +| `completeness`, `truncated` | the traversal's budget *and* the generation's own completeness | +| `omissions` | `provider_has_more`, `unresolved_entities`, `provider_warning`, `document_limit`, `provider_partial` | +| `documents[].confidence` | the provider's qualification of each relationship | +| `documents[].citations` | validated file/line references into the bound commit | +| `binding` | the authorization envelope, unchanged | + +Confidence maps the provider's own qualification onto the contract's vocabulary: + +| Provider evidence | Packet confidence | Meaning | +| --- | --- | --- | +| `extracted` | `extracted` | parsed from the source | +| `inferred` | `inferred` | derived, not read directly | +| `ambiguous` | `unknown` | resolved to more than one candidate | + +`ambiguous` also raises `unresolved_entities` on the packet, so a recipient sees +the uncertainty at the packet level and not only per document. A target name +that matches more than one definition does the same. + +A document's citations are both endpoints of the relationship it states plus, +where the provider recorded a call-site location and it resolves against the +bound commit, the edge's own — titled `call site: ` so a recipient +can tell it apart from either endpoint. A relationship whose call site carries +no location leaves the endpoint citations exactly as before; one whose stated +call site the bound commit does not carry — an untracked path, or a line past +the end of the file — is never presented as verified evidence: the citation is +dropped and the packet still raises `provider_warning`, the same signal an +unresolved endpoint raises. + +Document text is metadata about relationships — names, paths, relationship +kinds, hop counts — and never indexed content. Everything in it is already in +the citations beside it, so the prose adds no claim a recipient cannot check. + +## Required blocks, optional degrades + +Usability is settled before anything is read, against `graph_status`: a graph +that is absent, stale, partial, corrupt or oversized never reaches a traversal. +Neither outcome raises, because "no graph" is a normal state of an opt-in +feature. + +| Policy | Graph unusable | `dependent_work` | Exit code | +| --- | --- | --- | --- | +| `required: true` | `required_unavailable` | `paused` | 1 | +| `required: false` | `optional_unavailable` | `usable` | 0 | + +The words match `context_prepare`, so a caller branches on one vocabulary. +Optional unavailable context is not a degraded answer — there is no packet at +all, and the next action is to carry on with ordinary repository tools. + +## One packet, three recipients + +Claude, Codex and Devin receive the same approved bytes through the same +delivery path. The rendered evidence is identical for each: it names no +connection, no credential, no provider tool, and no local path, and a recipient +needs no graph, no provider install, and no Graphify credentials of its own. +Authorization is unchanged — a recipient the envelope does not name is still +refused at load. + +## Command + +``` +code-mower context-graph query --question impact --target parse_config \ + --authorization AUTH.json [--packet-out PACKET.json] \ + [--revision REV] [--depth N] [--node-budget N] [--json] +``` + +`--authorization` names a JSON file carrying the connection envelope, the +policy, the repository and the work item. It is read from a file the operator +names rather than discovered: this command mints evidence for named recipients, +and which recipients those are is an authorization decision that belongs to the +connection, not to a query. + +Standard output is metadata only — counts, states, the bound revision and +generation, and the omission codes. The evidence itself goes to the private file +named by `--packet-out`, created `0600`, or nowhere at all. A destination that +already exists is replaced rather than reopened: a creation mode binds only a +file the open creates, so writing into an existing world-readable path would put +the evidence behind whatever permissions that path already carried. The packet +is written to a freshly created private sibling and renamed over the +destination, which is also atomic — a reader never sees a half-written packet, +and a failed write leaves the previous file untouched. + +## Guided sessions + +`context-graph query` is the standalone verb. The same graph is also reachable +from the ordinary guided route, by registering it as a context connection: + +``` +code-mower context-graph connect --connection local-graph \ + --repository owner/repo --recipient claude:builder --recipient codex:reviewer +code-mower context-graph connection-status --connection local-graph +code-mower context-graph disconnect --connection local-graph +``` + +A local connection has no principal, no workspace, and no credential. Nothing +is written to the OS credential vault, no browser opens, and no endpoint is +contacted; the connection's whole state is one checkout and the repositories +and recipients the operator approved for it. `session context prepare` then +reaches it through the shared packet store, with the same protected handle, the +same authorization scope, the same work item and recipient contract, and the +same attachment and delivery path an organization connection uses: + +``` +code-mower session context prepare SESSION.json \ + --question impact --query-stdin # stdin names the symbol or path +``` + +`--question` is this connection's retrieval source, and the query names the +symbol or repository-relative path. Both are explicit: a graph answers about a +named target, and guessing one out of a work item's prose would produce +confident evidence about whatever happened to match. `--question` defaults to +`symbol`. + +What replaces the credential is the graph. Authorization is re-derived from +current local state on every load and every replay — never cached — and the +envelope carries the **published generation** as its `generation`. Two rules +then fall out of the shared packet contract rather than out of new checks: + +- A **rebuilt** graph publishes a new generation, so a packet bound to the old + one no longer matches its envelope and is refused at load. +- A **moved `HEAD`** makes the published generation stale for that revision, so + authorization fails outright and nothing is delivered. + +The revision both rules are evaluated against is the **consuming** one — the +commit the checkout doing the work is at — and it is never defaulted. A guided +session reads it from the directory it is preparing work in; `context fetch` +reads it from the directory it was run in. Neither falls back to resolving +`HEAD` in the checkout the connection was registered against, because that is a +different directory that moves on its own: a graph built for commit A would +otherwise answer work at commit B. A caller that is not in a Git checkout at all +names no revision, and a repository graph refuses rather than guessing one. +Organization context is unaffected: its sources carry document versions that +have no reason to equal a code commit. + +Required context that is refused pauses the dependent work; optional context +degrades and the session continues with ordinary repository tools. Claude, +Codex and Devin receive byte-identical approved evidence, and no recipient +needs the provider, the pin, or any Graphify tool to read it. + +## Boundary + +This change adds no dependency, no background service, and no mandatory +indexing step. Nothing is selected by default: a graph is indexed only when an +operator builds one, and reached from a guided session only when an operator +connects one. Hosted +Graphify, semantic or model-based extraction, clustering, watchers, and provider +API keys remain out of scope and separate decisions. diff --git a/docs/graphify-evaluation.md b/docs/graphify-evaluation.md index 22af78fc..f046edcd 100644 --- a/docs/graphify-evaluation.md +++ b/docs/graphify-evaluation.md @@ -174,8 +174,15 @@ lifecycle in outside every checkout, a manifest that binds full commit and tree with build time, opt-in acquisition with no default dependency, and a `partial` completeness state that refuses to read a fast incremental repeat as a complete -graph. Conditions 4 and 6 belong to the retrieval adapter, which does not exist -yet. +graph. + +Conditions 4 and 6 are implemented by the query and packet adapter in +[Local repository graph: bounded queries and context packets](context-graph-queries.md) +(issue #914): symbol-first, relationship-filtered traversals with explicit +depth and node budgets, truncation reported as `truncated` plus a +`provider_has_more` omission rather than hidden, and a packet whose citations +are validated against the bound commit's tracked tree before any of it reaches +a recipient. ## Boundary diff --git a/src/code_mower/context_audit.py b/src/code_mower/context_audit.py index 067272bc..dedc7f25 100644 --- a/src/code_mower/context_audit.py +++ b/src/code_mower/context_audit.py @@ -69,8 +69,12 @@ def finish(self, *, head, prose): if not review_matches(marker(current, review=True), current, head=head): return False if self.delivery is not None: + # The immutable audited target head, not a control checkout's + # revision: the checkout running this audit may differ from + # the code being reviewed. verified = deliver(self.store, current["revision"], repository=self.repository, - pr=self.pr, head=head, recipient=self.recipient, current=current, backend=self.backend) + pr=self.pr, head=head, recipient=self.recipient, current=current, + consuming_revision=head, backend=self.backend) if verified.text != self.delivery.text: return False save_feedback(self.store, verified, self.recipient.split(":")[0], prose) @@ -109,8 +113,12 @@ def prepare(*, repository, pr, head, host, authorities, fetch_comments, state.ready = review_matches(marker(current, review=True), current, head=head) return state state.store = store if store is not None else ContextStore(state_dir) + # ``head`` is the immutable review-target revision this audit is for, + # never derived from ``repo_path`` or ``Path.cwd()``: the audit + # control checkout may differ from the selected review target. state.delivery = deliver(state.store, current["revision"], repository=repository, - pr=pr, head=head, recipient=state.recipient, current=current, backend=backend) + pr=pr, head=head, recipient=state.recipient, current=current, + consuming_revision=head, backend=backend) state.ready = True except (ContextError, OSError, ValueError, RuntimeError, TypeError): pass diff --git a/src/code_mower/context_command.py b/src/code_mower/context_command.py index dcde73b3..87175104 100644 --- a/src/code_mower/context_command.py +++ b/src/code_mower/context_command.py @@ -14,7 +14,7 @@ from .claude_audit_pr import _decision_authorities_for_repo from .context_contract import ContextError, ContextRequest, _object, normalize_policy from .context_delivery import SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, attach, deliver, read_binding, render_evidence -from .context_packets import load_authorized +from .context_packets import consuming_revision, load_authorized from .context_store import ContextStore, strict_json from .provider_runners import fetch_issue_comments, fetch_pull_request, post_pr_comment from .provider_runners.github_auth import resolve_github_token_from_env_or_gh @@ -63,7 +63,8 @@ def main(argv=None): if args.recipient not in SUPPORTED_RECIPIENTS or args.recipient.endswith(":reviewer"): raise ContextError("independent reviewers consume an attached review revision") packet = load_authorized(store, args.connection, args.packet, spec["policy"], - ContextRequest(spec["repository"], spec["work_item"], args.recipient)) + ContextRequest(spec["repository"], spec["work_item"], args.recipient, + consuming_revision(args.repo_path))) print(render_evidence(packet, args.packet), end="") return 0 if args.command in ('deliver', 'feedback') and (args.connection or args.request_stdin): @@ -107,9 +108,14 @@ def publish(metadata): # The current trusted gate independently rejects policy downgrade. publish(metadata) else: + # The actual consuming checkout, derived from --repo-path, not + # the remote PR head: a checkout at a different commit, or a + # directory that is not a Git checkout at all, must fail + # before any evidence is reserved or published. metadata = attach(store, args.connection, spec["packet"], spec["policy"], ContextRequest(spec["repository"], spec["work_item"], args.host + ":orchestrator"), - pr=spec["pr"], head=head, publish=publish) + pr=spec["pr"], head=head, publish=publish, + consuming_revision=consuming_revision(args.repo_path)) print(json.dumps({"status": "attached", **metadata}, sort_keys=True)) return 0 if not args.revision or args.packet: @@ -120,8 +126,13 @@ def publish(metadata): if current is None: raise ContextError("no trusted current context input is declared") head = fetch_pull_request(binding["repository"], binding["pr"], token=token)["head"]["sha"] + # The actual consuming checkout, derived from --repo-path, not the + # remote PR head: a checkout at a different commit, or a directory + # that is not a Git checkout at all, must fail before any evidence or + # feedback is produced. delivery = deliver(store, args.revision, repository=binding["repository"], pr=binding["pr"], head=head, - recipient=args.recipient, current=current) + recipient=args.recipient, current=current, + consuming_revision=consuming_revision(args.repo_path)) if args.command == "feedback": feedback = delivery.binding["feedback"].get(args.reviewer) if feedback is None: diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index fc8eab5b..fd7075a9 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -6,6 +6,7 @@ import uuid from dataclasses import dataclass, field +from . import context_graph_connection as graph_connection from . import context_review from .context_connections import _state from .context_contract import ContextError, ContextRequest, ValidatedPacket, _identifier, _object, _text, normalize_policy @@ -75,11 +76,17 @@ def read_binding(store, revision): return _binding(lookup.artifact("d-" + revision).read()) -def _packet_for_binding(store, binding, recipient, *, backend=None): +def _packet_for_binding(store, binding, recipient, *, backend=None, revision=None): if recipient not in SUPPORTED_RECIPIENTS: raise ContextError("this participant cannot consume private context in this release") + # ``revision`` is the caller's actual consuming checkout revision, or an + # already-verified immutable review-target head; it is never defaulted to + # the head this binding happens to have been published for. A repository + # connection re-derives its authorization from exactly that commit and + # fails closed when it is missing; an organization connection ignores it. packet = load_authorized(store, binding["connection"], binding["handle"], binding["policy"], - ContextRequest(binding["repository"], binding["work_item"], recipient), backend=backend) + ContextRequest(binding["repository"], binding["work_item"], recipient), backend=backend, + revision=revision) if packet.sha256 != binding["packet_sha256"]: raise ContextError("context evidence changed; attach the new input and review again") return packet @@ -95,6 +102,7 @@ def reserve_attachment( pr, head, revision=None, + consuming_revision=None, backend=None, ): """Reserve one unpublished binding before any remote publication. @@ -102,11 +110,28 @@ def reserve_attachment( A caller-supplied revision lets a guided session persist its intent before touching GitHub and resume that exact intent after a crash. Repeating the same reservation is idempotent; a conflicting reuse fails closed. + + ``revision`` here is that attachment handle, not a Git revision. ``head`` + is attachment metadata: the PR head the caller read from the trusted + remote. ``consuming_revision`` is the caller's actual consuming checkout + revision, or ``None`` when the caller cannot name one; it is never + defaulted to ``head``, because a caller that knows only the trusted + remote head, not the local checkout doing the work, must say so + explicitly rather than let a repository-kind connection be silently + authorized against a revision it never held. Repository-kind evidence is + authorized against that consuming revision and separately checked against + ``head``, so a packet prepared while the checkout sat at commit A cannot + be attached to a PR whose head is commit B, and a checkout that has + itself moved to commit B cannot attach evidence for commit A. """ if request.recipient not in SUPPORTED_RECIPIENTS or not request.recipient.endswith(":orchestrator"): raise ContextError("an approved orchestrator must attach context") policy = normalize_policy(policy) - packet = load_authorized(store, name, handle, policy, request, backend=backend) + packet = load_authorized( + store, name, handle, policy, + ContextRequest(request.repository, request.work_item, request.recipient, consuming_revision), + backend=backend, revision=consuming_revision, + ) payload = packet.private_payload() revision = revision or uuid.uuid4().hex _handle(revision) @@ -114,8 +139,26 @@ def reserve_attachment( "state": "available", "expires_at": payload["binding"]["expires_at"]}) render_evidence(packet, handle) with store.locked(name) as locked: - state = _state(locked.read(), name) - if state["state"] != "verified" or state["generation"] != payload["binding"]["generation"]: + saved = locked.read() + if graph_connection.is_graph(saved): + state = graph_connection.saved_state(saved, name) + # The local graph's "authorization changed" is a rebuild: the + # published generation is what a packet binds, so a graph rebuilt + # between preparation and attachment fails the same check a revoked + # organization authorization does. ``consuming_revision`` cannot be + # ``None`` here: a graph connection already refused the load above + # when it was missing. + generation = graph_connection.current_generation(state, root=store.root, revision=consuming_revision) + # Stated here as well as enforced on the load, because this is the + # line an attachment is read off: repository evidence describes one + # commit's code, and the commit this PR is at is the only one it may + # be attached to. + if payload["source_revision"] != head: + raise ContextError("context evidence is not bound to the current pull request head") + else: + state = _state(saved, name) + generation = state["generation"] + if state["state"] != "verified" or generation != payload["binding"]["generation"]: raise ContextError("context authorization changed before attachment") index_file, index = _index(locked) entry = next((item for item in index["entries"] if item["handle"] == handle), None) @@ -162,6 +205,16 @@ def mark_published(store, name, revision): def _remove_attachment(store, name, handle, revision, *, published): + """Remove one identity-checked binding, idempotent for its own interrupted cleanup. + + The index is written without ``revision`` before the artifact is deleted, + so a crash between those two writes leaves an artifact whose index entry + already omits it. Retrying with the exact same ``handle``/``revision`` + recognizes that state -- the artifact's own binding still names them -- + and finishes deleting the artifact rather than reporting an inconsistent + index. A missing or mismatched identity, or a missing index entry, is + never treated as that same interrupted cleanup and still fails closed. + """ _handle(handle) _handle(revision) with store.locked(name) as locked: @@ -175,12 +228,16 @@ def _remove_attachment(store, name, handle, revision, *, published): index_file.write(index) return binding = _binding(saved) + if binding["handle"] != handle or binding["revision"] != revision: + raise ContextError("context attachment index is inconsistent") if binding["published"] and not published: raise ContextError("published context attachment cannot be abandoned") - if entry is None or revision not in entry.setdefault("deliveries", []): + if entry is None: raise ContextError("context attachment index is inconsistent") - entry["deliveries"].remove(revision) - index_file.write(index) + deliveries = entry.setdefault("deliveries", []) + if revision in deliveries: + deliveries.remove(revision) + index_file.write(index) artifact.delete() @@ -194,14 +251,17 @@ def retire_attachment(store, name, handle, revision): _remove_attachment(store, name, handle, revision, published=True) -def attach(store, name, handle, policy, request: ContextRequest, *, pr, head, publish, backend=None): +def attach(store, name, handle, policy, request: ContextRequest, *, pr, head, publish, + consuming_revision=None, backend=None): """Publish a new input revision before any participant can use that binding. ``publish`` is a trusted runtime callback for the selected repository/PR, never provider code. A failed publication leaves the local binding unusable. + ``consuming_revision`` is forwarded to ``reserve_attachment`` unchanged. """ metadata = reserve_attachment( - store, name, handle, policy, request, pr=pr, head=head, backend=backend, + store, name, handle, policy, request, pr=pr, head=head, + consuming_revision=consuming_revision, backend=backend, ) revision = metadata["revision"] try: @@ -222,7 +282,16 @@ class Delivery: binding: dict = field(repr=False) -def deliver(store, revision, *, repository, pr, head, recipient, current, backend=None): +def deliver(store, revision, *, repository, pr, head, recipient, current, consuming_revision=None, backend=None): + """Replay one published binding for an approved recipient. + + ``consuming_revision`` is the caller's actual consuming checkout revision, + or an already-verified immutable review-target head; ``None`` means the + caller cannot name one. It is never defaulted to ``head`` here: a caller + that knows only the trusted remote head, not the local checkout doing the + work, must say so explicitly rather than let a repository-kind connection + be silently authorized against a revision it never held. + """ binding = read_binding(store, revision) try: current = context_review.validate(dict(current)) @@ -233,7 +302,7 @@ def deliver(store, revision, *, repository, pr, head, recipient, current, backen raise ContextError("context input is missing, unpublished, or no longer current") if not context_review.review_matches(context_review.marker(current, review=True), current, head=head): raise ContextError("context input is unavailable or expired") - packet = _packet_for_binding(store, binding, recipient, backend=backend) + packet = _packet_for_binding(store, binding, recipient, backend=backend, revision=consuming_revision) return Delivery(dict(current), render_evidence(packet, binding["handle"]), binding) diff --git a/src/code_mower/context_graph.py b/src/code_mower/context_graph.py index 07f5390e..ce1f124c 100644 --- a/src/code_mower/context_graph.py +++ b/src/code_mower/context_graph.py @@ -31,7 +31,7 @@ #: these means the graph escaped the immutable checkout it was asked to index. #: Compared case-folded: on a case-insensitive filesystem (APFS and NTFS by #: default) ``.GIT/config`` names the same directory as ``.git/config``. -_EXCLUDED_ROOTS = frozenset({".git", ".graph", ".graphify", ".code-mower"}) +_EXCLUDED_ROOTS = frozenset({".git", ".graph", ".graphify", "graphify-out", ".code-mower"}) def _names_private_state(parts: Iterable[str]) -> bool: diff --git a/src/code_mower/context_graph_command.py b/src/code_mower/context_graph_command.py index a78108f7..58eb3ee4 100644 --- a/src/code_mower/context_graph_command.py +++ b/src/code_mower/context_graph_command.py @@ -16,15 +16,24 @@ import argparse import json +import os import sys from pathlib import Path +from . import context_graph_connection as connection from . import context_graph_lifecycle as lifecycle +from . import context_graph_query as query from .context_contract import ContextError -from .context_store import strict_json +from .context_store import ContextStore, strict_json MAX_PIN_BYTES = 8192 +#: The authorization envelope and policy a packet binds to. Read from a file +#: the operator names, never discovered: this command mints evidence for named +#: recipients, and which recipients those are is an authorization decision that +#: belongs to the connection, not to a query. +MAX_AUTHORIZATION_BYTES = 65_536 + def _load_pin(path: Path | None) -> lifecycle.GraphifyPin | None: if path is None: @@ -48,6 +57,63 @@ def _require_pin(path: Path | None) -> lifecycle.GraphifyPin: return pin +def _load_authorization(path: Path) -> dict: + """The connection envelope and policy one packet will bind to. + + Bounded at the stream for the same reason the pin is: a bound checked on + bytes already in memory is not a bound on what the file can cost to read. + """ + try: + with path.open("rb") as stream: + raw = stream.read(MAX_AUTHORIZATION_BYTES + 1) + except OSError: + raise ContextError("context authorization file is unreadable") from None + if len(raw) > MAX_AUTHORIZATION_BYTES: + raise ContextError("context authorization file exceeds its bound") + payload = strict_json(raw) + if not isinstance(payload, dict) or set(payload) != {"connection", "policy", "repository", "work_item"}: + raise ContextError("context authorization must carry a connection, policy, repository and work item") + return payload + + +def _write_packet(destination: Path, packet: dict) -> None: + """Write one packet where only this operator can read it. + + ``0o600`` at creation rather than after: the delivery path refuses a packet + whose mode ever allowed anyone else, and a chmod after the fact is a window + in which it did. + + A destination that already exists is *replaced*, never reopened. The + creation mode an open passes is only honoured for a file the open creates, + so writing into an existing world-readable path would put the evidence + behind whatever permissions that path already carried -- which the delivery + contract then refuses, after the bytes are already readable. The packet is + therefore written to a freshly created private sibling and renamed over the + destination, which is also atomic: a reader never sees a half-written + packet, and a failed write leaves the previous file untouched. + """ + body = json.dumps(packet, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":")) + # ``O_EXCL`` so the mode below is the mode of a file this process created. + # The name is unique per process rather than random: this directory is the + # operator's own, and a leftover from a crashed run must not be adopted. + staging = destination.with_name(f".{destination.name}.{os.getpid()}.partial") + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(body) + stream.flush() + os.fsync(stream.fileno()) + os.replace(staging, destination) + except BaseException: + # The evidence never survives a failed write under a name anyone asked + # for, and never under the staging name either. + try: + os.unlink(staging) + except OSError: + pass + raise + + def _emit(payload: dict, *, as_json: bool, text: str) -> None: if as_json: print(json.dumps(payload, indent=2, sort_keys=True)) @@ -66,12 +132,20 @@ def main(argv=None) -> int: status = sub.add_parser("status", help="Report whether the published generation may be used") remove = sub.add_parser("remove", help="Delete this checkout's private local graph state") doctor = sub.add_parser("doctor", help="Check the local graph posture without building anything") + ask = sub.add_parser("query", help="Answer one bounded question and emit a revision-bound packet") + # The guided route. ``query`` is the standalone verb an operator drives by + # hand; these three register the same graph as an ordinary context + # connection, so ``session context prepare``/``deliver`` can reach it + # through the shared packet store without a Coworker account or SDK. + join = sub.add_parser("connect", help="Register this checkout's graph as a local context connection") + leave = sub.add_parser("disconnect", help="Disable the connection and drop the packets it authorized") + linked = sub.add_parser("connection-status", help="Report the connection and the graph behind it") - for command in (build, refresh, status, remove, doctor): + for command in (build, refresh, status, remove, doctor, ask, join, leave, linked): command.add_argument("--repo-path", type=Path, default=Path.cwd(), help="Checkout to bind") command.add_argument("--state-dir", type=Path, help="Private state root; defaults to the context store") command.add_argument("--json", action="store_true", help="Emit a machine-readable summary") - for command in (build, refresh, status, doctor): + for command in (build, refresh, status, doctor, ask): command.add_argument("--revision", default="HEAD", help="Revision to bind, for example a commit or tag") for command in (build, refresh, doctor): command.add_argument("--pin-file", type=Path, help="JSON file naming one exact provider release") @@ -82,6 +156,22 @@ def main(argv=None) -> int: status.add_argument("--allow-partial", action="store_true", help="Treat a provider-declared partial build as usable") remove.add_argument("--show-local-paths", action="store_true", help="Include the private state path in output") + ask.add_argument("--question", required=True, choices=query.QUESTIONS, help="Which bounded question to ask") + ask.add_argument("--target", required=True, help="Symbol name, or a repository-relative path") + ask.add_argument("--authorization", type=Path, required=True, + help="JSON file with the connection envelope, policy, repository and work item") + ask.add_argument("--packet-out", type=Path, + help="Write the private packet to this file, created 0600") + ask.add_argument("--depth", type=int, help="Traversal depth; defaults to the question's own ceiling") + ask.add_argument("--node-budget", type=int, default=query.DEFAULT_NODE_BUDGET, + help="Most relationships one answer may carry before it reports truncation") + linked.add_argument("--revision", default="HEAD", help="Revision the connection must currently bind") + for command in (join, leave, linked): + command.add_argument("--connection", required=True, help="Context connection name to register or inspect") + join.add_argument("--repository", required=True, action="append", metavar="OWNER/NAME", + help="Approved context repository; repeat for more than one") + join.add_argument("--recipient", required=True, action="append", metavar="HOST:ROLE", + help="Approved recipient, for example claude:builder; repeat for more than one") args = parser.parse_args(argv) try: @@ -137,6 +227,50 @@ def main(argv=None) -> int: # A non-current graph is a normal, reportable condition, not a # command failure; exit 1 so a script can branch on usability. return 0 if report.usable else 1 + if args.command == "query": + authorization = _load_authorization(args.authorization) + outcome = query.graph_context( + args.repo_path, + question=args.question, + target=args.target, + envelope=authorization["connection"], + policy=authorization["policy"], + context_repository=authorization["repository"], + work_item=authorization["work_item"], + root=args.state_dir, + revision=args.revision, + depth=args.depth, + node_budget=args.node_budget, + ) + if outcome.packet is not None and args.packet_out is not None: + _write_packet(args.packet_out, outcome.packet) + # Metadata only, here as everywhere in this command: the summary + # carries counts, states and the binding, and the evidence itself + # goes to the private file the operator named or nowhere at all. + lines = [f"Local graph query: {outcome.status}"] + lines.extend(f" {key}: {value}" for key, value in sorted(outcome.summary.items()) + if key not in ("schema", "status")) + _emit(outcome.summary, as_json=args.json, text="\n".join(lines) + "\n") + return outcome.exit_code + if args.command in ("connect", "disconnect", "connection-status"): + store = ContextStore(args.state_dir) + if args.command == "connect": + summary = connection.connect(store, args.connection, { + "repository_root": str(Path(args.repo_path).resolve()), + "repositories": list(args.repository), + "recipients": list(args.recipient), + }) + elif args.command == "disconnect": + summary = connection.disconnect(store, args.connection) + else: + summary = connection.status( + store, args.connection, root=args.state_dir, revision=args.revision, + ) + lines = [f"Local graph connection: {summary['status']}"] + lines.extend(f" {key}: {value}" for key, value in sorted(summary.items()) + if key not in ("schema", "status")) + _emit(summary, as_json=args.json, text="\n".join(lines) + "\n") + return 0 if summary.get("authorization", "available") == "available" else 1 if args.command == "remove": state = lifecycle.GraphStateRoot(args.repo_path, root=args.state_dir) path = str(state.path) if args.show_local_paths else None diff --git a/src/code_mower/context_graph_connection.py b/src/code_mower/context_graph_connection.py new file mode 100644 index 00000000..8f50af15 --- /dev/null +++ b/src/code_mower/context_graph_connection.py @@ -0,0 +1,308 @@ +"""Graphify as a local repository connection for the shared context store. + +The query module in ``context_graph_query`` answers one bounded question and +mints one packet. This module is what makes that answer reachable from the +ordinary guided route -- ``session context prepare``, ``deliver``, and the +reuse and attachment that follow -- without a second packet store, a second +delivery contract, or a second set of recipients. + +Three things are deliberately absent, because a local graph has none of them: +there is no principal, no workspace, and no credential. Connecting names a +checkout and the repositories and recipients an operator approves for it, and +that is the whole of the connection's state. Nothing here reads the OS +credential vault, opens a browser, or contacts a network endpoint. + +What replaces the credential is the graph itself. Authorization is not a saved +token that stays true until it expires; it is re-derived from current trusted +local state on every single load, from ``graph_status`` for the requested +revision, and the envelope it returns carries the *published generation* as its +``generation``. That one choice is what makes the freshness rules fall out of +the shared contract rather than out of new checks here: + +* A rebuilt graph publishes a new generation, so a packet minted against the + old one no longer matches the envelope, and ``load_packet`` refuses it. +* A moved ``HEAD`` makes ``graph_status`` report the published generation stale + for that revision, so authorization fails outright and nothing is delivered. + +A recipient therefore cannot be handed evidence from a graph that no longer +describes the code, and no recipient needs the provider, its pin, or any +Graphify tool to read what it is given. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping + +from . import context_graph_lifecycle as lifecycle +from . import context_graph_query as query +from .context_contract import ( + CAPABILITY_VERSION, CONNECTION_SCHEMA, ContextError, _identifier, _object, + _strings, _text, normalize_policy, validate_connection, +) +from .context_store import ContextStore + +#: Saved state for one local repository connection. Distinct from the private +#: Coworker connection schema so a store can hold both and every reader can +#: tell which contract it is looking at before it validates anything. +GRAPH_SCHEMA = "code_mower.contextLocalGraphConnection.v1" +PROVIDER = "graphify" +CONNECTION_KIND = "repository" + +#: Which bounded question a retrieval asks when the caller names none. The +#: guided route carries a free-text ``source`` for exactly this purpose. +DEFAULT_QUESTION = "symbol" + +#: How long one live authorization stands. Short on purpose: it bounds a packet +#: that was already minted, and the graph behind it is re-checked on every load +#: regardless, so a longer window would buy nothing and hide a stale build for +#: longer if a check were ever skipped. +AUTHORIZATION_SECONDS = 3600 + + +def is_graph(value: Any) -> bool: + """Whether saved connection state belongs to this contract. + + Cheap and structural, so a caller can branch before validating. A state + that claims this schema and then fails ``saved_state`` is an error, not a + reason to fall through to the organization path. + """ + return isinstance(value, Mapping) and value.get("schema") == GRAPH_SCHEMA + + +def connection_spec(value: Any) -> dict[str, Any]: + """The operator's approval: one checkout, and who may read answers from it.""" + spec = _object(value, {"repository_root", "repositories", "recipients"}) + root = Path(_text(spec["repository_root"], maximum=4096)) + if not root.is_absolute(): + raise ContextError("local context repository root must be absolute") + return { + "repository_root": str(root), + "repositories": list(_strings(spec["repositories"])), + "recipients": list(_strings(spec["recipients"])), + } + + +def saved_state(value: Any, name: str) -> dict[str, Any]: + if value is None: + raise ContextError("context connection is missing; run context-graph connect") + state = _object(value, {"schema", "connection", "provider", "kind", "state", + "repository_root", "repositories", "recipients"}) + if (state["schema"] != GRAPH_SCHEMA or state["connection"] != name + or state["provider"] != PROVIDER or state["kind"] != CONNECTION_KIND + or state["state"] not in {"verified", "disconnected"}): + raise ContextError("unsupported local graph context state; reconnect") + return {**state, **connection_spec({key: state[key] for key in + ("repository_root", "repositories", "recipients")})} + + +def _summary(state: Mapping[str, Any]) -> dict[str, Any]: + return {"schema": "code_mower.contextConnectionSummary.v1", "provider": PROVIDER, + "kind": CONNECTION_KIND, "status": state["state"], "search": "available", + "memory": "unavailable", "credential_storage": "none"} + + +def connect(store: ContextStore, name: str, spec: Any) -> dict[str, Any]: + """Record one approved checkout. No credential is created or stored.""" + name = _identifier(name) + spec = connection_spec(spec) + # Resolving now rather than at every load: the identity a packet binds is + # the one the operator approved, and a root that later becomes a symlink + # elsewhere must not quietly redirect the evidence. + root = lifecycle.checkout_root(Path(spec["repository_root"])) + with store.locked(name, timeout_seconds=1) as locked: + old = locked.read() + if old is not None and not is_graph(old): + raise ContextError("this connection name already names a different context provider") + if old is not None and saved_state(old, name)["state"] != "disconnected": + raise ContextError("connection already exists; disconnect before changing its checkout or scope") + if old is not None: + # A reconnect retries packet cleanup under this same lock before + # authorizing anything again, even when the disconnect that + # preceded it already reported the cleanup complete: a packet or + # delivery binding that survived that attempt must not become + # authorized again just because the graph and scope are unchanged. + try: + from .context_packets import purge_connection + purge_connection(locked) + except Exception as exc: + raise ContextError("pending packet cleanup failed; reconnect refused") from exc + state = {"schema": GRAPH_SCHEMA, "connection": name, "provider": PROVIDER, + "kind": CONNECTION_KIND, "state": "verified", "repository_root": str(root), + "repositories": spec["repositories"], "recipients": spec["recipients"]} + locked.write(state) + return _summary(state) + + +def disconnect(store: ContextStore, name: str) -> dict[str, Any]: + """Disable the connection first, then drop the packets it authorized.""" + with store.locked(name) as locked: + state = {**saved_state(locked.read(), name), "state": "disconnected"} + locked.write(state) + packet_cleanup = "complete" + try: + from .context_packets import purge_connection + purge_connection(locked) + except Exception: + packet_cleanup = "needs_attention" + return {**_summary(state), "packet_cleanup": packet_cleanup} + + +def status(store: ContextStore, name: str, *, root: Path | None = None, + revision: str = "HEAD") -> dict[str, Any]: + """Report the connection and the graph behind it, without minting evidence.""" + with store.locked(name) as locked: + state = saved_state(locked.read(), name) + report = lifecycle.graph_status(Path(state["repository_root"]), root=root, revision=revision) + return {**_summary(state), "graph": report.shareable_summary(), + "authorization": "available" if state["state"] == "verified" and report.usable + else "unavailable"} + + +def _published(state: Mapping[str, Any], *, root: Path | None, + revision: str) -> lifecycle.GenerationStatus: + if state["state"] != "verified": + raise ContextError("local graph context is disconnected; reconnect before using it") + report = lifecycle.graph_status(Path(state["repository_root"]), root=root, revision=revision) + if not report.usable or report.manifest is None: + # The state word is the lifecycle's own -- ``stale``, ``partial``, + # ``absent`` -- and it is the only detail worth carrying: it says + # whether to rebuild, refresh, or build for the first time. + raise ContextError( + "local graph is not current for this revision (" + report.state + "); rebuild it" + ) + return report + + +def current_generation(state: Mapping[str, Any], *, root: Path | None = None, + revision: str = "HEAD") -> str: + """The published generation a packet must still match to be usable.""" + manifest = _published(state, root=root, revision=revision).manifest + assert manifest is not None # a usable status always carries one + return manifest.generation + + +def authorized_revision(locked, name: str, *, root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> tuple[dict[str, Any], str]: + """One envelope, and the commit the graph is bound to for ``revision``. + + The commit is returned rather than re-derived by the caller because it is + the same ``graph_status`` read that authorized the load: asking twice would + rehash the artifact and, worse, could answer differently across a rebuild, + so the envelope and the revision a packet is checked against would then come + from two different observations of the graph. + + ``revision`` is the *consuming* revision -- the commit whose work the + evidence is for -- and not merely the registered checkout's ``HEAD``. A + graph that was not built from exactly that commit resolves ``stale`` in + ``_published`` and never reaches a recipient. + """ + current = now or datetime.now(timezone.utc) + state = saved_state(locked.read(), name) + manifest = _published(state, root=root, revision=revision).manifest + assert manifest is not None + return _envelope(state, name, manifest, current), manifest.commit + + +def authorize_locked(locked, name: str, *, root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> dict[str, Any]: + """Mint one envelope from current local state, under the caller's lock. + + This is the local counterpart of the organization connection's online + refresh, and it is deliberately the same shape: every load and every replay + calls it again, and none of them may reuse a previous answer. What it + checks is not a token but the graph -- present, complete, and binding the + requested revision -- and what it publishes as ``generation`` is the + graph's, so the shared packet contract does the rest. + """ + return authorized_revision(locked, name, root=root, revision=revision, now=now)[0] + + +def _envelope(state: Mapping[str, Any], name: str, manifest: Any, + current: datetime) -> dict[str, Any]: + envelope = { + "schema": CONNECTION_SCHEMA, "capability_version": CAPABILITY_VERSION, + "connection": name, "provider": PROVIDER, "kind": CONNECTION_KIND, + "generation": manifest.generation, "state": "verified", + "identity": {"repository_root": state["repository_root"]}, + "repositories": list(state["repositories"]), "recipients": list(state["recipients"]), + "expires_at": (current + timedelta(seconds=AUTHORIZATION_SECONDS)).isoformat(), + # No memory, and a real revision binding: the packet names the commit + # the graph was built from, so a recipient asking about another one + # resolves ``stale`` without decoding the evidence. + "capabilities": {"search": True, "memory": False, "revision_binding": True}, + } + return dict(validate_connection(envelope, now=current)) + + +def question_and_target(spec: Mapping[str, Any]) -> tuple[str, str]: + """Read one bounded question out of the shared retrieval request. + + The guided request carries a free-text ``query`` and an optional ``source``. + For a graph those are the target and the question, and both are explicit on + purpose: this connection answers about a named symbol or a repository- + relative path, and guessing one out of a work item's prose would produce + confident evidence about whatever happened to match. + """ + question = spec.get("source") or DEFAULT_QUESTION + if question not in query.QUESTIONS: + raise ContextError( + "local graph source must name a question: " + ", ".join(query.QUESTIONS) + ) + target = _text(spec["query"], maximum=2000) + return question, target + + +def retrieve(state: Mapping[str, Any], spec: Mapping[str, Any], *, envelope: Mapping[str, Any], + root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> dict[str, Any]: + """Answer once from the published generation, or say why there is no answer. + + ``graph_context`` returns rather than raises when the graph cannot answer, + because "no graph" is a normal state of an opt-in feature. The packet store + is not a place where that distinction survives -- a reservation either + produces a packet or it does not -- so the unavailable outcomes become the + error the store already knows how to unwind, carrying the reason forward. + """ + question, target = question_and_target(spec) + policy = normalize_policy(spec["policy"]) + if policy is None: + raise ContextError("context policy is not configured") + outcome = query.graph_context( + Path(state["repository_root"]), + question=question, + target=target, + envelope=envelope, + policy=policy, + context_repository=spec["repository"], + work_item=spec["work_item"], + root=root, + revision=revision, + now=now, + ) + if outcome.packet is None: + raise ContextError( + "local graph context is unavailable (" + str(outcome.summary.get("reason", "unknown")) + ")" + ) + return outcome.packet + + +__all__ = ( + "AUTHORIZATION_SECONDS", + "CONNECTION_KIND", + "DEFAULT_QUESTION", + "GRAPH_SCHEMA", + "PROVIDER", + "authorize_locked", + "authorized_revision", + "connect", + "connection_spec", + "current_generation", + "disconnect", + "is_graph", + "question_and_target", + "retrieve", + "saved_state", + "status", +) diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index cb77bda7..0f638d91 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -58,19 +58,22 @@ import re import io import secrets +import shlex import shutil import signal import socket import stat +import struct import subprocess import sys import tarfile import tempfile import time +import unicodedata import uuid from dataclasses import dataclass from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence from .context_contract import ContextError, _identifier, _text, _timestamp @@ -113,15 +116,32 @@ _REGULAR_MODES = frozenset({"100644", "100755"}) _SKIPPED_MODES = {"120000": "symlink", "160000": "submodule"} -#: Where the provider keeps its own index state. Both names are on the -#: excluded-roots list in ``context_graph``, and a repository is free to track -#: either of them -- a committed ``.graph/`` is somebody else's graph, or an -#: earlier incremental cache of this one. Neither may be materialized: the +#: Where the pinned provider actually writes. Read off ``graphify/paths.py`` at +#: the evaluated pin rather than assumed: ``GRAPHIFY_OUT`` defaults to the +#: literal ``graphify-out`` and every output path is built from it, so an +#: unmodified ``extract`` run in the materialized copy leaves +#: ``graphify-out/graph.json`` and ``graphify-out/manifest.json`` beneath the +#: scan target. The environment override that constant reads is deliberately +#: *not* honoured here: the child is given a fixed environment, and an output +#: root this adapter did not choose is a root it cannot bound to the copy. +_PROVIDER_OUTPUT_DIRECTORY = "graphify-out" + +#: The document the ``--no-cluster`` branch dumps, and the provider's own record +#: of which inputs it processed. Exactly these two names are read; a run that +#: leaves something else has not produced the evidence this adapter classifies. +_PROVIDER_GRAPH_NAME = "graph.json" +_PROVIDER_MANIFEST_NAME = "manifest.json" + +#: Where the provider keeps its own index state or output. All three names are +#: on the excluded-roots list in ``context_graph``, and a repository is free to +#: track any of them -- a committed ``.graph/`` is somebody else's graph, a +#: committed ``graphify-out/`` is an earlier build's published output, and +#: ``.graphify/`` is an incremental cache. None may be materialized: the #: provider would then resume from a cache built over content this build never #: saw, and the adapter would collect tracked repository bytes as if the #: provider had just produced them, binding stale contents to a fresh commit. #: Matched at any depth and case-folded, for the same reasons ``.git`` is. -_PROVIDER_STATE_DIRECTORIES = (".graphify", ".graph") +_PROVIDER_STATE_DIRECTORIES = (_PROVIDER_OUTPUT_DIRECTORY, ".graphify", ".graph") _PROVIDER_STATE_ROOTS = frozenset(name.casefold() for name in _PROVIDER_STATE_DIRECTORIES) #: The evidence contract's excluded roots, bound rather than copied. One module @@ -1329,6 +1349,13 @@ class IndexRequest: #: directory the environment points at but the sandbox does not expose is a #: provider that cannot start. writable: tuple[Path, ...] = () + #: The census the copy was materialized from -- the denominator completeness + #: is measured against. Carried on the request rather than re-read from the + #: copy after the run, because by then the provider has written into that + #: tree: the question is what this build *gave* the provider, and only the + #: census is immutable evidence of that. A request without one cannot be + #: classified as complete, which is the safe direction. + census: TrackedCensus | None = None @dataclass(frozen=True) @@ -1337,6 +1364,12 @@ class IndexResult: completeness: str = COMPLETE indexed_files: int = 0 + #: Inputs this provider classified as code and then deterministically could + #: not extract -- no wired extractor, or an extractor that declined by + #: design. Read bytes, no contribution; reported rather than counted as + #: indexed, and distinct from the census entries a build declines to + #: materialize at all. + unsupported_inputs: int = 0 notes: tuple[str, ...] = () @@ -1592,6 +1625,531 @@ def _refuse_broad_readable(readable: Iterable[str], *, repository: Path) -> None ) +#: Mach-O magics, and the ``struct`` byte order each one means. A Mach-O image +#: declares its own order in its first four bytes; the swapped spellings are how +#: a big-endian image announces itself to a little-endian reader. Both widths +#: are listed with the size of the header that follows, because the load +#: commands this reads begin directly after it. +_MACHO_MAGICS: Mapping[bytes, tuple[str, int]] = { + b"\xcf\xfa\xed\xfe": ("<", 32), # 64-bit, little endian + b"\xce\xfa\xed\xfe": ("<", 28), # 32-bit, little endian + b"\xfe\xed\xfa\xcf": (">", 32), # 64-bit, big endian + b"\xfe\xed\xfa\xce": (">", 28), # 32-bit, big endian +} + +#: The 64-bit thin header's width, which is also what says an image's pointers +#: -- and so the multiple its load-command sizes must be -- are eight bytes. +_MACHO_HEADER_BYTES_64 = 32 + +#: A load command's own header: the command and its size. +_MACHO_COMMAND_HEADER_BYTES = 8 + +#: A universal ("fat") archive: a count of architecture records, each naming the +#: offset and size of a real Mach-O image inside the same file. A python.org +#: interpreter ships these; a Homebrew one does not. +#: +#: The two spellings are the same header written in the two byte orders, and the +#: magic is what says which: ``FAT_MAGIC`` reads as ``0xCAFEBABE`` big-endian, so +#: the count and the architecture records that follow it are big-endian too; +#: ``FAT_CIGAM`` is those four bytes reversed, and its fields are reversed with +#: them. Reading a ``FAT_CIGAM`` header as big-endian -- which is what this did +#: -- turns a count of 2 into 33_554_432 and a slice offset into a number with +#: no relation to the file, so the container was not being decoded at all. +_MACHO_FAT_ORDERS: Mapping[bytes, str] = { + b"\xca\xfe\xba\xbe": ">", # FAT_MAGIC + b"\xbe\xba\xfe\xca": "<", # FAT_CIGAM +} + +#: One architecture record: cputype, cpusubtype, offset, size, align. +_MACHO_FAT_ARCH_BYTES = 20 + +#: The 64-bit universal header (``FAT_MAGIC_64``) is deliberately absent. Its +#: records are a different width with 64-bit offsets, so admitting it would mean +#: a second parse; an unrecognized container is refused rather than guessed at, +#: and no runtime this boundary has had to derive ships one. + +#: The load commands that name a library the image will make dyld find. Weak, +#: re-exported and upward links are included: a weak dependency that *is* +#: installed is still opened, and an image that re-exports another still loads +#: it. ``LC_REQ_DYLD`` is the high bit these commands carry. +_MACHO_DYLIB_COMMANDS = frozenset({0x0C, 0x8000_0018, 0x8000_001F, 0x8000_0023}) + +#: How much of an image's load-command block this will read. The block is a +#: header, not the image; anything past this bound is not a Mach-O this module +#: is prepared to reason about. +_MAX_MACHO_COMMAND_BYTES = 4 * 1024 * 1024 + +#: How many architecture slices a universal archive may declare. +_MAX_MACHO_ARCHITECTURES = 32 + +#: How many files the runtime scan will look at, and how many libraries it will +#: add. Both are bounds on somebody else's install, which is the same reason +#: every other foreign file this module reads is bounded: a provider install is +#: not this repository's to trust about its own size. +_MAX_SCANNED_IMAGES = 20_000 +_MAX_LINKED_LIBRARIES = 256 + +#: A shared library that is larger than this is not one; refusing is the honest +#: answer rather than mapping an arbitrary file into the child's view. +_MAX_LINKED_LIBRARY_BYTES = 512 * 1024 * 1024 + +#: File names that are never a Mach-O image, so the scan does not open them. +#: Purely an optimization -- the magic is what decides -- but it is what keeps +#: the walk over a populated ``site-packages`` cheap. +_NOT_MACHO_SUFFIXES = frozenset( + { + ".py", ".pyc", ".pyi", ".pyx", ".txt", ".md", ".rst", ".json", ".toml", + ".yaml", ".yml", ".cfg", ".ini", ".h", ".hpp", ".c", ".cpp", ".html", + ".css", ".js", ".png", ".jpg", ".svg", ".gif", ".pdf", ".zip", ".gz", + ".whl", ".pem", ".crt", ".dist-info", ".egg-info", ".a", ".la", + } +) + +#: Directories the scan does not descend into: build caches and vendored +#: sources, none of which hold an image the child loads. +_NOT_MACHO_DIRECTORIES = frozenset({"__pycache__", ".git", "include", "man", "doc", "docs"}) + + +@dataclass(frozen=True) +class _MachoSlice: + """One structurally validated Mach-O image inside a file. + + Produced only by :func:`_macho_slices`, and only after that function has + established that every field below names a region the file actually carries. + Holding the located load-command region rather than its bytes keeps the peak + cost of reading a universal archive one slice's block rather than all of + them at once. + """ + + order: str + """The byte order this image declared in its own magic.""" + + filetype: int + """What the image says it is: ``MH_DYLIB``, ``MH_EXECUTE``, and so on.""" + + ncmds: int + """How many load commands the region carries. Validated to walk exactly.""" + + commands_offset: int + """Absolute offset in the file of the first load command.""" + + sizeofcmds: int + """Size of the whole load-command region, bounded and known to be present.""" + + +def _macho_slices(path: Path) -> tuple[_MachoSlice, ...] | None: + """Every Mach-O image in ``path``, or ``None`` if it is not a Mach-O file. + + The one structural parse: what a container *is* is decided here, and both + the dependency derivation and the exposure refusal read its answer rather + than each re-deriving a weaker version of it. ``None`` means this module + could not read the file as a Mach-O at all, which covers a file that is not + one, a truncated one, and a malformed one -- a universal header declaring + slices the file does not carry, slices that overlap each other or the + architecture table, a load-command region that runs past its slice, or a + command chain that does not walk to exactly the size the header declared. + + Nothing here is inferred from a name, a size or a suffix. Every bound is + checked against the file's own length, taken from the open descriptor so + that what is measured is what is read. The whole parse is bounded before it + allocates: at most :data:`_MAX_MACHO_ARCHITECTURES` slices, each with at + most :data:`_MAX_MACHO_COMMAND_BYTES` of load commands, read one at a time. + + A header alone is not enough to make this judgement, which is why it is made + here rather than at a magic and a ``filetype`` field. Those four bytes and + that one integer are the cheapest thing in the file for a provider that + writes its own linker input to reproduce over arbitrary operator-owned + bytes; the structure behind them is not. + """ + try: + with path.open("rb") as stream: + size = os.fstat(stream.fileno()).st_size + magic = stream.read(4) + order = _MACHO_FAT_ORDERS.get(magic) + if order is not None: + return _macho_fat_slices(stream, order=order, size=size) + if magic not in _MACHO_MAGICS: + return None + located = _macho_slice(stream, offset=0, limit=size) + return None if located is None else (located,) + except (OSError, ValueError, struct.error): + return None + + +def _macho_fat_slices( + stream: io.BufferedReader, *, order: str, size: int +) -> tuple[_MachoSlice, ...] | None: + """The validated slices of a universal archive, or ``None`` if malformed. + + Every slice must parse, not merely one: the child is the provider's + interpreter, whose architecture is not necessarily this one, so a fat file + admitted on the strength of a readable first slice would be admitting + whatever the rest of it holds. + """ + raw = stream.read(4) + if len(raw) != 4: + return None + count = struct.unpack(f"{order}I", raw)[0] + if not 1 <= count <= _MAX_MACHO_ARCHITECTURES: + return None + table_end = 8 + _MACHO_FAT_ARCH_BYTES * count + if table_end > size: + # The header promises more architecture records than the file carries. + return None + extents: list[tuple[int, int]] = [] + for _ in range(count): + record = stream.read(_MACHO_FAT_ARCH_BYTES) + if len(record) != _MACHO_FAT_ARCH_BYTES: + return None + # cputype, cpusubtype, offset, size, align + _, _, offset, length, _ = struct.unpack(f"{order}5I", record) + if length <= 0 or offset < table_end or offset + length > size: + # A slice that starts inside the header this is reading, or that + # runs off the end of the file, describes a file other than this + # one. Both fields came out of the file, so both are bounded + # 32-bit values and the sum cannot overflow a Python int. + return None + extents.append((offset, length)) + ordered = sorted(extents) + for (offset, length), (next_offset, _) in zip(ordered, ordered[1:], strict=False): + if offset + length > next_offset: + # Overlapping slices make "which image is this" ambiguous, and an + # ambiguous container is not one to answer a trust question from. + return None + slices = [] + for offset, length in extents: + located = _macho_slice(stream, offset=offset, limit=length) + if located is None: + return None + slices.append(located) + return tuple(slices) + + +def _macho_slice(stream: io.BufferedReader, *, offset: int, limit: int) -> _MachoSlice | None: + """One Mach-O image of ``limit`` bytes at ``offset``, fully validated. + + ``limit`` is the slice's own extent -- the whole file for a thin image, the + architecture record's declared size for a fat one -- and every read is held + inside it. A slice may not reach past itself into another slice's bytes to + satisfy its header. + """ + stream.seek(offset) + magic = stream.read(4) + order_and_header = _MACHO_MAGICS.get(magic) + if order_and_header is None: + return None + order, header_size = order_and_header + if limit < header_size: + return None + header = stream.read(header_size - 4) + if len(header) != header_size - 4: + return None + # cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags[, reserved] + _, _, filetype, ncmds, sizeofcmds, _ = struct.unpack(f"{order}6I", header[:24]) + if sizeofcmds > _MAX_MACHO_COMMAND_BYTES: + return None + if header_size + sizeofcmds > limit: + # The declared load-command region does not fit in the image that + # declared it. + return None + if ncmds * _MACHO_COMMAND_HEADER_BYTES > sizeofcmds: + # Every load command carries at least its own command and size, so a + # region this small cannot hold the count claimed over it. + return None + block = stream.read(sizeofcmds) + if len(block) != sizeofcmds: + return None + pointer = 8 if header_size == _MACHO_HEADER_BYTES_64 else 4 + if not _macho_commands_walk(block, order=order, ncmds=ncmds, pointer=pointer): + return None + return _MachoSlice( + order=order, + filetype=int(filetype), + ncmds=int(ncmds), + commands_offset=offset + header_size, + sizeofcmds=int(sizeofcmds), + ) + + +def _macho_commands_walk(block: bytes, *, order: str, ncmds: int, pointer: int) -> bool: + """Whether ``block`` is exactly ``ncmds`` well-formed load commands. + + Exactly: the chain must consume the region the header declared, with nothing + left over and nothing missing. ``sizeofcmds`` is defined as the total size + of all the load commands, so a chain that stops short leaves bytes in a + region that is supposed to be nothing but commands, and a chain that would + run past has already lied about one command's size. Either way the image is + not describing itself, and this refuses rather than reading as far as it can + and treating the prefix as the truth. + + ``pointer`` is the image's pointer width, which is the multiple dyld + requires each ``cmdsize`` to be: 8 for a 64-bit image, 4 for a 32-bit one. + """ + position = 0 + for _ in range(ncmds): + if position + _MACHO_COMMAND_HEADER_BYTES > len(block): + return False + _, size = struct.unpack_from(f"{order}2I", block, position) + if size < _MACHO_COMMAND_HEADER_BYTES or size % pointer: + return False + if position + size > len(block): + return False + position += size + return position == len(block) + + +def _macho_dylib_names(path: Path) -> tuple[str, ...]: + """Every library path a Mach-O image at ``path`` asks dyld to load. + + Read out of the image's own load commands rather than out of ``otool``: + deriving the boundary must not itself depend on a developer tool being + installed, and a parse that reads a bounded header is a smaller thing to + trust than a subprocess. A file that is not a Mach-O -- which is almost + everything under an install prefix -- costs four bytes and returns nothing. + + The union over a universal archive's slices rather than the slice matching + this process: the child is the provider's interpreter, whose architecture is + not necessarily this one, and every slice's dependencies are paths on the + same host. + + A file this cannot read structurally contributes nothing. An unreadable or + malformed image says nothing about what the runtime needs; the build still + fails if it was a library the provider loads, and it fails as dyld naming + the image rather than as this module guessing at one. + """ + slices = _macho_slices(path) + if not slices: + return () + names: dict[str, None] = {} + try: + with path.open("rb") as stream: + for located in slices: + stream.seek(located.commands_offset) + block = stream.read(located.sizeofcmds) + if len(block) != located.sizeofcmds: + return () + _collect_dylib_names(block, order=located.order, ncmds=located.ncmds, into=names) + except (OSError, ValueError, struct.error): + return () + return tuple(names) + + +def _collect_dylib_names( + block: bytes, *, order: str, ncmds: int, into: dict[str, None] +) -> None: + """The ``LC_LOAD_DYLIB`` family in an already-validated command region.""" + position = 0 + for _ in range(ncmds): + command, size = struct.unpack_from(f"{order}2I", block, position) + if command in _MACHO_DYLIB_COMMANDS and size >= 24: + name_offset = struct.unpack_from(f"{order}I", block, position + 8)[0] + if 8 <= name_offset < size: + raw = block[position + name_offset : position + size] + name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") + if name: + into.setdefault(name, None) + position += size + + +#: The Mach-O file types dyld will open for an ``LC_LOAD_DYLIB``-family command: +#: ``MH_DYLIB``, and the ``MH_DYLIB_STUB`` a stripped SDK ships in its place. +#: Not ``MH_BUNDLE``, which is reached through ``dlopen`` rather than through +#: the load commands this derivation reads, and not an executable, an object +#: file, a core dump or a kernel extension -- none of which is a dependency an +#: image's load command can legitimately name. +_MACHO_DYLIB_FILETYPES = frozenset({0x6, 0x9}) + + +def _macho_filetypes(path: Path) -> tuple[int, ...] | None: + """Every slice's declared Mach-O ``filetype``, or ``None`` for a non-Mach-O. + + The two answers are different refusals for the caller, so they are kept + apart rather than collapsed into a falsehood. ``None`` is a file whose + container this module could not read as a Mach-O -- not one at all, + truncated, or structurally malformed. A tuple is a file whose structure it + did validate, and whose own declaration of what it is the caller then holds + to the shared-library rule. + """ + slices = _macho_slices(path) + return None if slices is None else tuple(located.filetype for located in slices) + + +def _scan_images(root: Path, *, budget: list[int]) -> Iterator[Path]: + """Regular files under ``root`` that could be Mach-O images, within a budget. + + ``budget`` is shared across every root of one derivation, so the cost is a + property of the whole runtime rather than of each prefix in it. Symlinks are + not followed during the walk: an install that links a directory elsewhere is + reached through whatever named it, and following would let one link turn a + narrow prefix into an unbounded traversal. + """ + for parent, directories, files in os.walk(root, followlinks=False): + directories[:] = [name for name in directories if name not in _NOT_MACHO_DIRECTORIES] + for name in files: + if budget[0] <= 0: + return + if any(name.endswith(suffix) for suffix in _NOT_MACHO_SUFFIXES): + continue + budget[0] -= 1 + yield Path(parent) / name + + +def _trusted_library(path: Path) -> bool: + """Could only a trusted account have put this library where the child reads it? + + The same question :func:`_trusted_launcher` asks of a sandbox launcher, and + for the same reason: a library the provider maps executable inside the + boundary is code, and a file -- or a directory above it -- that some other + account may write is a file somebody else chooses the contents of. Asked of + the *resolved* path, so a link's own spelling is not what is trusted. + """ + trusted = {0, os.geteuid()} + for current in (path, *path.parents): + try: + entry = os.lstat(current) + except OSError: + return False + writable = bool(entry.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + if writable and stat.S_ISDIR(entry.st_mode) and entry.st_mode & stat.S_ISVTX: + # A sticky shared directory -- ``/tmp`` and the per-user temporary + # directories under it. Another account may create its own entries + # there and may not touch this one, which is the whole point of the + # bit, so the ancestry it provides is not an account boundary this + # has to refuse. The file itself is still held to the rule. + continue + if entry.st_uid not in trusted or writable: + return False + return True + + +def _linked_runtime_libraries( + roots: Sequence[Path], *, covered: Sequence[Path], repository: Path +) -> tuple[str, ...]: + """Shared libraries the exposed runtime links to from outside the exposure. + + A pinned provider's install and the base interpreter it was created from are + exposed as prefixes, and that was taken to be the whole runtime. It is not, + on a host whose interpreter was installed by a package manager: CPython's + ``_ssl`` extension is linked against an OpenSSL that lives under the + manager's own prefix, not under the interpreter's, and Graphify imports + ``ssl`` during start-up even for a code-only extraction. Under the + filesystem boundary that library is simply absent, so the provider aborted + at import time -- which is a missing runtime dependency, not an argument for + giving the child a network or a wider filesystem. + + So the dependency is *derived* rather than named. Every Mach-O image inside + the exposure is read for the libraries it asks dyld to load, and each one + that is not already covered is resolved and added as a single file. Newly + added libraries are read in turn, so a transitive dependency -- ``libssl`` + needing ``libcrypto`` -- is reached without either being written down here. + + What is added is the library file, never the directory holding it: exposing + ``/opt/homebrew/opt/openssl@3/lib`` is a package manager's prefix, and + exposing the manager's ``etc`` or ``var`` beside it is the operator data + this boundary exists to withhold. Every added path is put through the same + ownership and broad-exposure refusals as any other exposure, and a path that + is not a bounded regular file -- or that is not a structurally valid Mach-O + declaring itself a shared library -- is refused rather than exposed on the + strength of an image having named it. That last check is what keeps the + dependency *names*, which come out of somebody else's image, from choosing + which of the operator's files this boundary exposes, and it is the whole + container rather than a magic and a ``filetype`` field that has to hold up: + see :func:`_macho_slices`. + + Linux is unchanged: an ELF runtime's libraries live under the ``/lib`` and + ``/usr/lib`` directories the read-only runtime already names, and this + derivation reads Mach-O images, of which such a host has none. + """ + if sys.platform != "darwin": + return () + system = tuple(Path(os.path.realpath(path)) for path in _SYSTEM_READ_PATHS) + boundaries = [*system, *(Path(os.path.realpath(root)) for root in covered)] + added: dict[str, None] = {} + pending = [Path(os.path.realpath(root)) for root in roots] + budget = [_MAX_SCANNED_IMAGES] + while pending: + current = pending.pop(0) + images = _scan_images(current, budget=budget) if current.is_dir() else iter((current,)) + for image in images: + for name in _macho_dylib_names(image): + if not name.startswith("/"): + # ``@rpath``, ``@loader_path`` and ``@executable_path`` are + # resolved by dyld against the image itself, so they name + # something inside the exposure already. + continue + referenced = Path(name) + resolved = Path(os.path.realpath(referenced)) + if any(_under(resolved, boundary) for boundary in boundaries): + continue + if str(resolved) in added: + continue + if not resolved.exists(): + # A weak dependency the host does not have installed. If it + # was a required one the provider fails at launch, as dyld + # naming the library it could not find. + continue + _refuse_linked_library(resolved, repository=repository) + if len(added) >= _MAX_LINKED_LIBRARIES: + raise ContextError( + "the local graph provider's runtime links to more shared libraries " + "outside its install than this boundary is willing to expose; " + "no generation was published" + ) + added[str(resolved)] = None + if str(referenced) != str(resolved): + # Both spellings, for the same reason the executable has + # two: a bind-mount boundary names a literal destination, + # and dyld opens the path the image wrote down. + added.setdefault(str(referenced), None) + pending.append(resolved) + return tuple(added) + + +def _refuse_linked_library(resolved: Path, *, repository: Path) -> None: + """Refuse a derived dependency that is not a library this may expose.""" + _refuse_broad_exposure(resolved, repository=repository) + try: + info = os.lstat(resolved) + except OSError: # pragma: no cover - the caller has just seen it exist + raise ContextError( + "a shared library the local graph provider's runtime links to could not be " + "read while deriving the containment boundary; no generation was published" + ) from None + if not stat.S_ISREG(info.st_mode): + raise ContextError( + "the local graph provider's runtime links to something that is not a regular " + "file; refusing to expose it to the sandbox" + ) + if info.st_size > _MAX_LINKED_LIBRARY_BYTES: + raise ContextError( + "a shared library the local graph provider's runtime links to is implausibly " + "large for one; refusing to expose it to the sandbox" + ) + if not _trusted_library(resolved): + raise ContextError( + "a shared library the local graph provider's runtime links to is writable by " + "an account other than yours or root, so what the provider would load inside " + "the sandbox is not what this host installed; no generation was published" + ) + # Last, and the only check that reads the candidate's contents rather than + # its metadata. Ownership and size say who wrote a file and how big it is, + # not what it is, and the name comes out of a load command in somebody + # else's image -- so without this the provider chooses which of the + # operator's files the boundary exposes by writing its own linker input. + # The candidate must say it is a shared library in its own header. + filetypes = _macho_filetypes(resolved) + if filetypes is None: + raise ContextError( + "the local graph provider's runtime names a dependency that is not a readable " + "Mach-O image; refusing to expose it to the sandbox" + ) + if any(filetype not in _MACHO_DYLIB_FILETYPES for filetype in filetypes): + raise ContextError( + "the local graph provider's runtime names a dependency that is a Mach-O image " + "but not a shared library; refusing to expose it to the sandbox" + ) + + def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: """The install the pinned provider needs to be readable, and nothing beside it. @@ -1640,7 +2198,16 @@ def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: # out of *this* environment's own ``pyvenv.cfg`` rather than taken from the # interpreter Code Mower happens to be running under, which is a different # installation whenever the provider was pinned with a different Python. - return (*spellings, str(root), *_provider_base_prefixes(root, repository=repository)) + prefixes = (str(root), *_provider_base_prefixes(root, repository=repository)) + # Last, and derived from the prefixes rather than added to them: a prefix is + # not the whole runtime on a host whose interpreter links against libraries + # a package manager keeps somewhere else. + libraries = _linked_runtime_libraries( + [Path(prefix) for prefix in prefixes], + covered=[Path(prefix) for prefix in prefixes], + repository=repository, + ) + return (*spellings, *prefixes, *libraries) #: Where an installed distribution records its own identity (PEP 376). The @@ -1869,37 +2436,212 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: #: The subcommand the evaluated release exposes, recorded in #: ``docs/graphify-evaluation.md``: the clean-room run indexed with #: ``extract --code-only --no-cluster --max-workers 4``. There is no -#: ``--source``/``--output`` pair to hand it; ``extract`` reads the directory -#: it is run in and writes its state beside those sources, which is why the -#: child's working directory is the materialized copy and why the adapter -#: collects an artifact afterwards rather than naming one up front. +#: ``--source``/``--output`` pair to hand it; ``extract`` writes its state +#: beside the sources it was pointed at, which is why the child's working +#: directory is the materialized copy and why the adapter collects an artifact +#: afterwards rather than naming one up front. _PROVIDER_EXTRACT = "extract" -#: The provider's own record of what it processed. Completeness is read from -#: here, never inferred from an exit status: the clean-room run recorded 54 -#: manifest entries requeued by a repeat that exited zero in 1.63 s. -_PROVIDER_REPORT_NAMES = ("manifest.json", "index.json", "report.json") - -#: Counters whose presence above zero means the provider did not finish. Any -#: one of them, not all: a report that admits requeued entries is a partial -#: build however healthy the rest of it looks. -_INCOMPLETE_COUNTERS = ("requeued", "pending", "failed", "errors", "incomplete") - -#: Where the provider reports how many files it actually indexed. -_INDEXED_COUNTERS = ("indexed_files", "code_files", "files", "entries") - -#: An affirmative claim that the run finished, in either shape a report can -#: carry one. Nothing else counts: an empty object, or one whose schema this -#: adapter does not recognize, says nothing about completion and is therefore -#: not evidence of it. -_COMPLETION_FLAGS = ("complete", "completed", "finished") -#: Narrow on purpose: a field that names the run's state, not one that might -#: carry a path or a message, so an unrecognized value here is a real -#: non-completion rather than an adapter that read the wrong field. -_COMPLETION_STATUS_FIELDS = ("status", "state") -_COMPLETION_STATUS_VALUES = frozenset( - {"complete", "completed", "success", "succeeded", "ok", "finished", "done"} +#: The scan target, which the pinned CLI requires and does not default. +#: +#: This module used to launch ``extract`` with the options alone, on the reading +#: that a subcommand which writes beside its sources must also discover them +#: from the working directory. The pinned CLI does not: it takes the target as +#: the first positional after the subcommand, decides it has one only when that +#: argument does not begin with ``-``, and exits 1 with ``must specify a path to +#: scan or a --postgres DSN`` when it does not. Every real build therefore +#: failed before extraction, and the failure arrived as the generic non-zero +#: refusal rather than as anything naming the omission. +#: +#: ``.`` rather than the source root's absolute path: the child's working +#: directory is already the materialized copy, so the relative spelling names +#: exactly the tree this build means and names nothing about where that tree +#: sits on the host. It must be passed *between* the subcommand and the options +#: -- the CLI reads ``sys.argv[2]`` and nothing later -- and it is a path, so a +#: bare ``.`` can never be mistaken for a flag the way a caller-supplied string +#: could. +_PROVIDER_SCAN_TARGET = "." + +#: The extensions the pinned provider's own ``detect.CODE_EXTENSIONS`` treats as +#: code, transcribed from the hash-verified 0.9.58 wheel. This is the +#: denominator's definition and it has to be the provider's, not a plausible +#: one: a build is complete when every input the provider itself would dispatch +#: was processed, and holding it to every tracked documentation file instead +#: would make a correct run permanently partial. Nothing outside this set is +#: counted against the run -- those inputs are deterministically not code to +#: this pin, so they are skipped rather than missing. +_PROVIDER_CODE_EXTENSIONS = frozenset({ + ".py", ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".ejs", + ".ets", ".go", ".rs", ".java", ".groovy", ".gradle", ".cpp", ".cc", ".cxx", + ".c", ".h", ".hpp", ".cu", ".cuh", ".metal", ".rb", ".rake", ".swift", + ".kt", ".kts", ".cs", ".scala", ".php", ".lua", ".luau", ".toc", ".zig", + ".ps1", ".psm1", ".psd1", ".ex", ".exs", ".m", ".mm", ".ml", ".mli", ".jl", + ".vue", ".svelte", ".astro", ".dart", ".v", ".sv", ".svh", ".sql", ".r", + ".f", ".F", ".f90", ".F90", ".f95", ".F95", ".f03", ".F03", ".f08", ".F08", + ".pas", ".pp", ".dpr", ".dpk", ".lpr", ".inc", ".dfm", ".lfm", ".lpk", + ".sh", ".bash", ".json", ".tf", ".tfvars", ".hcl", ".dm", ".dme", ".dmi", + ".dmm", ".dmf", ".sln", ".slnx", ".csproj", ".fsproj", ".vbproj", ".xaml", + ".razor", ".cshtml", ".cls", ".trigger", ".lisp", ".cl", ".lsp", ".asd", + ".robot", ".resource", +}) + +#: Names the pinned ``detect.classify_file`` routes to code by *filename*, +#: ahead of every suffix class, because ``manifest_ingest`` parses them +#: deterministically (``PACKAGE_MANIFEST_NAMES``, compared lower-cased against +#: the basename). Suffix membership alone misses all of them -- ``.yml``, +#: ``.toml``, ``.mod`` and ``.xml`` are not in ``CODE_EXTENSIONS`` -- so a +#: denominator built from extensions would let a package manifest the provider +#: failed on drop out of the coverage question entirely. +_PROVIDER_PACKAGE_MANIFEST_NAMES = frozenset({ + "apm.yml", "apm.yaml", "pyproject.toml", "cargo.toml", "go.mod", "pom.xml", +}) + +#: The one compound suffix the pin tests before the simple one. ``.blade.php`` +#: already ends in a code extension, so this only decides *which* extractor the +#: pin uses; it is named here because the classification below is meant to be +#: readable against ``classify_file``'s own order rather than to be minimal. +_PROVIDER_COMPOUND_CODE_SUFFIX = ".blade.php" + +#: ``detect._SHEBANG_CODE_INTERPRETERS``: the interpreters that make an +#: *extensionless* tracked file code to this pin. ``classify_file`` reaches +#: this branch before any extension test, so a CLI entry point with no suffix +#: is dispatched exactly like a ``.py`` file and belongs in the denominator. +_PROVIDER_SHEBANG_CODE_INTERPRETERS = frozenset({ + "python", "python3", "python2", + "ruby", "perl", "node", "nodejs", + "bash", "sh", "dash", "zsh", "fish", "ksh", "tcsh", + "lua", "php", "julia", "Rscript", +}) + +#: ``extract._SHEBANG_DISPATCH``: the subset of the above that the pin has an +#: extractor for. The remainder (``perl``, ``fish``, ``tcsh``, ``Rscript``) is +#: classified as code and then deterministically contributes nothing. +_PROVIDER_SHEBANG_EXTRACTORS = frozenset({ + "python", "python2", "python3", + "bash", "sh", "dash", "zsh", "ksh", + "node", "nodejs", "ruby", "lua", "php", "julia", +}) + +#: The extensions in the pin's ``CODE_EXTENSIONS`` with no entry in +#: ``extract._DISPATCH`` -- the exact set difference of the two staged tables. +#: The pin warns about these itself (#1689: "classified as code but graphify +#: has no AST extractor for their language"), dispatches them, and stamps them, +#: because ``_get_extractor`` returning ``None`` short-circuits to +#: ``{"nodes": [], "edges": []}`` with neither an ``error`` nor a ``skipped`` +#: marker, and the CLI's failed-source rule only clears rows for the two cases +#: that carry one. They are counted and reported, never silently folded into +#: the files this build says were indexed. +_PROVIDER_UNSUPPORTED_EXTENSIONS = frozenset({".ejs", ".ets", ".r"}) + +#: ``.m`` is the one suffix whose dispatch the pin decides from the *bytes* +#: rather than the table: it is Objective-C or MATLAB/Octave, and +#: ``_get_extractor`` returns ``None`` for a ``.m`` carrying no Objective-C +#: directive (#1702) rather than force-parsing MATLAB through the ObjC grammar. +#: The static table difference above cannot see that, so without this branch a +#: MATLAB file would be counted as a file this build indexed on the strength of +#: a stamped row alone -- and the row *is* stamped, because a ``None`` extractor +#: short-circuits to ``{"nodes": [], "edges": []}`` carrying neither marker the +#: failed-source rule looks for. It is code either way, so it stays in the +#: denominator; what is decided here is only whether it contributed anything. +_PROVIDER_OBJC_AMBIGUOUS_SUFFIX = ".m" + +#: ``extract._OBJC_HEADER_MARKERS``: the Objective-C-only directives the pin +#: sniffs for, and the window it sniffs in (``_is_objc_header`` slices the first +#: 256 KiB). A marker past that window is invisible to the pin too, so reading +#: exactly that much decides the same way while staying bounded. +_PROVIDER_OBJC_MARKERS = ( + b"@interface", b"@protocol", b"@implementation", b"@import", b"#import", ) +_OBJC_PROBE_BYTES = 256 * 1024 + +#: How much of an extensionless input is read to find its shebang. The pin +#: reads the same 256 bytes and keeps only the first line. +_SHEBANG_PROBE_BYTES = 256 + +#: A manifest row's content hash, as the pin computes it: ``_md5_file`` streams +#: the file and returns a hex digest, or the empty string when the read failed. +#: So a well-formed 32-character digest is the provider's own statement that it +#: read those bytes, and anything else -- blank, short, uppercase, non-string -- +#: is a row that proves nothing about the file it names. +_MANIFEST_HASH = re.compile(r"[0-9a-f]{32}\Z") + +#: The row fields the pinned ``save_manifest`` writes. Read as a shape check +#: only: a mapping missing them is not the document this adapter can classify. +_MANIFEST_ROW_FIELDS = ("mtime", "seen", "ast_hash", "semantic_hash") + +#: How much of a materialized input is hashed at a time while deriving what the +#: provider's row for it must say. Bounded by the census, which is already +#: bounded by ``MAX_TRACKED_BYTES``, so this only bounds resident memory. +_MANIFEST_DIGEST_CHUNK_BYTES = 1024 * 1024 + + +def _materialized_digest(path: Path) -> str: + """The pin's own content digest of one materialized input, or ``""``. + + ``_md5_file`` in the pinned 0.9.58 wheel streams the file and returns the + MD5 hex digest of its bytes -- the same digest the fresh contained + extraction's manifest carried for both Python inputs of the public fixture, + matching their exact bytes. So this is not a re-implementation of the + provider's AST work: it is the one thing the provider's row is a statement + *about*, computed here so the statement can be checked rather than believed. + + ``""`` for anything that cannot be read as a regular file, which is a + refusal rather than a pass: a row this build cannot check is a row it cannot + count. + """ + digest = hashlib.md5(usedforsecurity=False) + try: + if path.is_symlink() or not path.is_file(): + return "" + with path.open("rb") as stream: + while True: + chunk = stream.read(_MANIFEST_DIGEST_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + except OSError: + return "" + return digest.hexdigest() + + +def _materialized_digests( + source_root: Path, + census: TrackedCensus | None, + inputs: _ProviderInputs | None = None, +) -> dict[str, str] | None: + """What every eligible code input's manifest row must say, before the run. + + Taken from the materialized copy *before* the provider is launched, which is + the only moment those bytes are still exactly what this build gave it. After + the run the same tree also holds provider output, and a digest read then + would be checking the provider's manifest against whatever the provider left + behind. + + ``None`` when there is no census, because there is then nothing to enumerate + -- that case is already the partial one. An input the copy cannot be read + for is simply absent from the map, and ``_read_completeness`` keeps the run + partial for it rather than accepting the row unchecked. + + Keyed by the exact Git tree path, never by a folded spelling of it. Two + tracked names that differ only by Unicode normalization are two files with + two blobs, and a map keyed on their shared normal form holds one digest for + both -- so the expected bytes of one input would be read from the other's. + Normalization is a question about what the *provider* named its rows, and it + is asked once, in ``_read_completeness``, against this exact denominator. + """ + if census is None: + return None + if inputs is None: + inputs = _provider_inputs(census, source_root) + digests: dict[str, str] = {} + for path in inputs.dispatched: + host = _census_host_path(source_root, path) + if host is None: + continue + digest = _materialized_digest(host) + if digest: + digests[path] = digest + return digests def _refuse_pre_existing_provider_state(source_root: Path) -> None: @@ -1921,111 +2663,496 @@ def _refuse_pre_existing_provider_state(source_root: Path) -> None: ) -def _provider_state_directory(source_root: Path) -> Path: - """The state directory the provider wrote during this run. +def _provider_output_directory(source_root: Path) -> Path: + """The output root the provider wrote during this run. + + Exactly one name is collected -- the pin's own ``graphify-out`` -- and it is + resolved beneath the materialized copy, never from a path or an environment + variable a caller could supply. Only reachable after + ``_refuse_pre_existing_provider_state``, so what is found here was created + by the run that just finished. - Only reachable after ``_refuse_pre_existing_provider_state``, so whichever - of the two names is present was created by the run that just finished. + A second provider root beside it is refused rather than ignored. This + adapter cannot tell which of two roots a generation should be cut from, and + picking one would publish an artifact whose provenance is a guess; a + ``.graphify/`` that appeared next to ``graphify-out/`` also says the run did + something other than the single contained extraction that was launched. """ - for name in _PROVIDER_STATE_DIRECTORIES: - candidate = source_root / name - if candidate.is_dir() and not candidate.is_symlink(): - return candidate - raise ContextError("local graph provider wrote no index state; no generation was published") + directory = source_root / _PROVIDER_OUTPUT_DIRECTORY + competing = [ + name + for name in _PROVIDER_STATE_DIRECTORIES + if name != _PROVIDER_OUTPUT_DIRECTORY + and ((source_root / name).exists() or (source_root / name).is_symlink()) + ] + if competing: + raise ContextError( + "local graph provider wrote more than one output root; no generation was published" + ) + if directory.is_symlink() or not directory.is_dir(): + raise ContextError("local graph provider wrote no output; no generation was published") + graph = directory / _PROVIDER_GRAPH_NAME + if graph.is_symlink() or not graph.is_file(): + raise ContextError( + "local graph provider left no graph document; no generation was published" + ) + return directory -def _provider_report(state_directory: Path) -> Mapping[str, Any] | None: - """The provider's completion evidence, or ``None`` if it left none. +def _provider_manifest(output_directory: Path) -> Mapping[str, Any] | None: + """The provider's own record of what it processed, or ``None`` if unreadable. - Bounded at the stream, not after the fact: the report is provider output of - unknown size, and reading it whole to slice it afterwards would let it + Bounded at the stream, not after the fact: the manifest is provider output + of unknown size, and reading it whole to slice it afterwards would let it exhaust this process before any budget was consulted. Anything longer than a manifest is rejected outright rather than parsed from a prefix, which would be a different document than the one the provider wrote. """ - for name in _PROVIDER_REPORT_NAMES: - path = state_directory / name - if not path.is_file() or path.is_symlink(): - continue - try: - with path.open("rb") as stream: - raw = stream.read(MAX_MANIFEST_BYTES + 1) - except OSError: - return None - if len(raw) > MAX_MANIFEST_BYTES: - return None - try: - payload = json.loads(raw) - except ValueError: - return None - return payload if isinstance(payload, Mapping) else None - return None + path = output_directory / _PROVIDER_MANIFEST_NAME + if path.is_symlink() or not path.is_file(): + return None + try: + with path.open("rb") as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + except OSError: + return None + if len(raw) > MAX_MANIFEST_BYTES: + return None + try: + payload = json.loads(raw) + except ValueError: + return None + return payload if isinstance(payload, Mapping) else None -def _completion_claim(report: Mapping[str, Any]) -> bool | None: - """``True`` finished, ``False`` denied it, ``None`` said nothing either way.""" - claim: bool | None = None - for name in _COMPLETION_FLAGS: - value = report.get(name) - if value is True: - claim = True - elif value is False: - return False - for field in _COMPLETION_STATUS_FIELDS: - value = report.get(field) - if not isinstance(value, str): - continue - if value.strip().casefold() in _COMPLETION_STATUS_VALUES: - claim = True - else: - # A status the adapter does not recognize is not a completion. - return False - return claim +@dataclass(frozen=True) +class _ProviderInputs: + """How the pin would classify this census, decided before it is launched. + + ``dispatched`` is the denominator: every tracked path ``classify_file`` + would call code, in census order. ``unsupported`` is the subset of those the + pin then has no extractor for, which is a real and reportable outcome rather + than a failure. ``unclassified`` is everything the classification could not + decide -- an extensionless input the copy could not be read for, or a + shebang spelling this adapter refuses to guess at -- and it keeps the run + partial, because an input nobody can classify is an input nobody can say was + covered. + """ + dispatched: tuple[str, ...] = () + unsupported: frozenset[str] = frozenset() + unclassified: tuple[str, ...] = () + + +def _census_host_path(source_root: Path, path: str) -> Path | None: + """Where a census path lives in the materialized copy, or ``None``. + + ``read_tracked_census`` reads Git's own index paths, which are relative and + carry no traversal segment. Held to that here anyway, because this is the + only place a census path is turned back into a host path, and a join is not + the place to discover otherwise. + """ + parts = PurePosixPath(path).parts + if not parts or any(part in ("", ".", "..", "/") for part in parts): + return None + return source_root.joinpath(*parts) -def _indexed_count(report: Mapping[str, Any]) -> int | None: - """How many files the provider says it indexed, if it says at all.""" - for counter in _INDEXED_COUNTERS: - value = report.get(counter) - if isinstance(value, int) and not isinstance(value, bool) and value >= 0: - return value - return None +def _shebang_interpreter(path: Path) -> tuple[str | None, bool]: + """``(interpreter, resolved)`` for an extensionless input's first line. -def _read_completeness(report: Mapping[str, Any] | None) -> IndexResult: - """Classify a provider run from its own report, defaulting to partial. + ``(None, True)`` is a decision: there is no shebang, so ``classify_file`` + would not call this file code. ``(None, False)`` is a refusal: the bytes + could not be read, or the line is one of the ``env(1)`` spellings the pin + resolves through option parsing this adapter deliberately does not + reimplement (``-S``/``--split-string`` and friends). A refusal is carried as + *unclassified* rather than guessed either way, because guessing "not code" + would drop a real input out of the denominator and guessing "code" would + demand a row for a file the pin never dispatched. - Absent or unreadable evidence is *not* evidence of a complete build, and - neither is a readable report that says nothing. ``complete`` is reached - only by a report shaped the way this adapter understands one: an - affirmative completion claim, a count of what was indexed, and no counter - admitting work left over. An empty object, an unrecognized schema, and a - document that happens to parse all stay ``partial``, which - ``graph_status`` refuses by default. The provider owns no provenance (the - evaluation records this as the first product constraint), so that refusal - is the failure an operator can act on; silently calling it complete is the - one they cannot. + Only the simple, unambiguous ``env`` forms are resolved here: leading + ``NAME=value`` assignments followed by the interpreter, which is what a + tracked script ordinarily carries. + """ + try: + if path.is_symlink() or not path.is_file(): + return None, False + with path.open("rb") as stream: + head = stream.read(_SHEBANG_PROBE_BYTES) + except OSError: + return None, False + if not head.startswith(b"#!"): + return None, True + line = head.split(b"\n")[0].decode(errors="replace")[2:].strip() + try: + parts = shlex.split(line) + except ValueError: + return None, False + if not parts: + return None, True + interpreter = PurePosixPath(parts[0].replace("\\", "/")).name + if interpreter != "env": + return interpreter, True + for argument in parts[1:]: + if argument.startswith("-"): + # An option-carrying ``env`` line. The pin has a full parser for + # these; this one says so rather than pretending to. + return None, False + if "=" in argument: + continue + return PurePosixPath(argument.replace("\\", "/")).name, True + return None, True + + +def _objc_source(path: Path) -> tuple[bool, bool]: + """``(objective_c, resolved)`` for one materialized ``.m`` input. + + ``(False, True)`` is a decision: the bytes carry no Objective-C directive, + so the pin's ``_get_extractor`` returns ``None`` for this file and it is + dispatched, stamped, and contributes nothing. ``(False, False)`` is a + refusal: the copy could not be read here, so which way the pin decided is + unknown and the caller carries the input as unclassified rather than + guessing. The pin's own sniff answers ``False`` on a read error, but that is + a statement about *its* read; this adapter failing to read the same bytes + proves nothing about what the provider was shown. + """ + try: + if path.is_symlink() or not path.is_file(): + return False, False + with path.open("rb") as stream: + head = stream.read(_OBJC_PROBE_BYTES) + except OSError: + return False, False + return any(marker in head for marker in _PROVIDER_OBJC_MARKERS), True + + +def _provider_inputs( + census: TrackedCensus, source_root: Path | None = None +) -> _ProviderInputs: + """Classify the census the way the pinned ``detect.classify_file`` would. + + In its order, which is the part suffix membership gets wrong: a package + manifest is routed by *filename* before any extension is looked at, and an + extensionless file is routed by its *shebang* before the extension table is + reached at all. A denominator built from ``CODE_EXTENSIONS`` alone drops + both, so a ``pyproject.toml`` or a ``#!/usr/bin/env python3`` CLI the + provider failed on could be missing from the manifest while an unrelated + ``.py`` file let the run claim it was complete. + + Case is tried both ways for the extension test because the pin's set carries + both ``.f90`` and ``.F90``; matching the spelling first and the lower-cased + suffix second can only widen the denominator, which is the direction that + refuses rather than over-claims. + + ``source_root`` is the materialized copy, read *before* the provider is + launched -- the only moment those bytes are still exactly what this build + handed over. Without it no extensionless input and no ``.m`` can be + classified, so each of them is unclassified and the run stays partial. + + Two dispatch questions are answered from those bytes rather than from a + table: which interpreter an extensionless script names, and whether a ``.m`` + is Objective-C or MATLAB. The second decides only whether a code input had + an extractor at all, never whether it is code -- ``.m`` is in the pin's + extension table either way. + """ + dispatched: list[str] = [] + unsupported: set[str] = set() + unclassified: list[str] = [] + for entry in census.entries: + path = entry.path + pure = PurePosixPath(path) + name = pure.name.lower() + if name in _PROVIDER_PACKAGE_MANIFEST_NAMES or name.endswith( + _PROVIDER_COMPOUND_CODE_SUFFIX + ): + dispatched.append(path) + continue + suffix = pure.suffix + if not suffix: + host = _census_host_path(source_root, path) if source_root is not None else None + if host is None: + unclassified.append(path) + continue + interpreter, resolved = _shebang_interpreter(host) + if not resolved: + unclassified.append(path) + elif interpreter in _PROVIDER_SHEBANG_CODE_INTERPRETERS: + dispatched.append(path) + if interpreter not in _PROVIDER_SHEBANG_EXTRACTORS: + unsupported.add(path) + continue + if suffix in _PROVIDER_CODE_EXTENSIONS or suffix.lower() in _PROVIDER_CODE_EXTENSIONS: + dispatched.append(path) + if suffix.lower() in _PROVIDER_UNSUPPORTED_EXTENSIONS: + unsupported.add(path) + elif suffix.lower() == _PROVIDER_OBJC_AMBIGUOUS_SUFFIX: + # The one dispatch the pin decides from the bytes. Read from the + # same pre-launch copy every other classification here reads, so + # the answer is about what the provider was handed. + host = _census_host_path(source_root, path) if source_root is not None else None + objective_c, resolved = _objc_source(host) if host is not None else (False, False) + if not resolved: + unclassified.append(path) + elif not objective_c: + unsupported.add(path) + return _ProviderInputs(tuple(dispatched), frozenset(unsupported), tuple(unclassified)) + + +def _census_normalization_collisions(census: TrackedCensus) -> frozenset[str]: + """Normal forms that more than one distinct tracked path folds onto. + + Git's tree is a set of exact byte paths, and two of them can be canonically + equivalent without being equal: ``café.py`` spelled with U+00E9 and the same + name spelled ``e`` + U+0301 are two entries, two blobs and two files, and a + Linux checkout carries both at once. The manifest this build reads back is + keyed by name, so those two inputs are the one case where a name cannot + identify an input on its own. + + Taken from the whole census rather than from the dispatched subset, because + the collapse this exists to catch is a property of the tracked names, and an + eligible input colliding with a tracked file the pin would not dispatch is + the same ambiguity. The census is immutable evidence taken before the launch, + so a provider that folded the pair into one manifest row cannot hide that + they were two. Bounded by the census, which ``read_tracked_census`` has + already bounded. + """ + seen: dict[str, str] = {} + collided: set[str] = set() + for entry in census.entries: + folded = unicodedata.normalize("NFC", entry.path) + if seen.setdefault(folded, entry.path) != entry.path: + collided.add(folded) + return frozenset(collided) + + +def _manifest_keys_for( + path: str, rows: Mapping[str, Any], folded: Mapping[str, Sequence[str]] +) -> Sequence[str]: + """Which manifest keys could be a record of ``path``. + + Exact identity first: a key spelled exactly as the tracked path names one + input and no other, whatever else the manifest carries, so it is never + reached past. The normalized fallback exists for the ordinary macOS case, + where the copy's filesystem hands the provider a canonically equivalent + spelling of the one name it was given, and it answers only when exactly one + key folds onto that name -- more than one is two records this build cannot + attribute, and the caller keeps the run partial rather than picking one. """ - if report is None: - return IndexResult(completeness=PARTIAL, notes=("provider left no readable completion report",)) + if path in rows: + return (path,) + return folded.get(unicodedata.normalize("NFC", path), ()) + + +def _read_completeness( + manifest: Mapping[str, Any] | None, + census: TrackedCensus | None, + digests: Mapping[str, str] | None = None, + inputs: _ProviderInputs | None = None, +) -> IndexResult: + """Classify a provider run against its own manifest, defaulting to partial. + + The pinned ``save_manifest`` writes a flat mapping of repository-relative + POSIX path to ``{mtime, seen, ast_hash, semantic_hash}``. It is not a + completion report and carries no flag or count, so completeness is a + coverage question: did every input this pin would dispatch come back with a + hash proving the provider read its bytes? + + The denominator is the immutable materialized census classified the way + ``detect.classify_file`` classifies it -- filename-routed package manifests + first, then extensionless shebang scripts, then the extension table (see + ``_provider_inputs``). Inputs outside that classification are + deterministically not code to this pin and are skipped, not missing. + Everything else is counted, and a + file is processed only when its row carries a well-formed ``ast_hash`` *and + that hash is the digest of the bytes this build actually handed the + provider*. A well-formed digest alone says a hash-shaped string is present; + only the comparison says it is a hash of this input. Without it a row + carried over from another tree, another revision, or a resumed cache reads + as proof of work on bytes the provider was never shown -- and the digests + come from ``_materialized_digests``, taken before the launch, so they cannot + have been influenced by what the run wrote. A row whose digest disagrees, and + an input the copy could not be re-read for, both stay partial. + + What a stamped row is evidence *of* comes from the pin's own + post-extraction writer rule, staged in ``extract.py`` and ``cli.py``. After + the run, ``_failed_sources`` is assembled from the per-file results and the + CLI clears (``clear_ast``) exactly those rows; every other dispatched input + is stamped. A result lands in ``_failed_sources`` when it carries an + ``error``, or when its extractor produced zero nodes. It does **not** when: + + * ``_get_extractor`` returned ``None`` -- the file short-circuits to + ``{"nodes": [], "edges": []}`` with neither marker, so a code-classified + input the pin has no extractor for is stamped while contributing nothing + (the pin's own #1689 warning); or + * the extractor declined by design -- ``extractors/json_config`` returns a + ``skipped`` marker for data JSON and for a non-object root, and the CLI + skips those deliberately so they are not requeued forever (#2879). + + So a stamped, matching row proves the provider read those exact bytes and + did not fail on them. It does not prove nodes, and it is not read here as + if it did. The deterministically unsupported dispatch is counted and + reported separately (``unsupported_inputs``) rather than folded into + ``indexed_files``, and zero nodes for a file whose row is stamped is a + complete *read* of that file and nothing more. + + The boundary, plainly. ``classify_file`` is the eligibility oracle: what it + deterministically calls not-code -- every suffix outside its registry + included -- is not in the denominator and does not make a run partial, so + there is no separate count of it. ``unsupported_inputs`` counts the inputs + it *does* call code that then reach a dispatch the pin has no extractor for: + the static table difference, a code shebang with no ``_SHEBANG_DISPATCH`` + entry, and a ``.m`` whose bytes carry no Objective-C directive. Anything + else is partial: an eligible code input that failed, one whose postcondition + is unknown, one whose row disagrees with the bytes, and one whose zero-node + result cannot be told apart from a failure. + + Names, and the one place they are not identities. The denominator is keyed + by exact Git tree path, and so is the digest map: two tracked names that are + canonically equivalent without being equal are two entries with two blobs, + coexisting in any Linux checkout, and folding them together would let one + input's stamped row and expected digest answer for the other -- a blank row + for a failed extraction overwritten by a successful twin, reported as two + files indexed. Unicode normalization survives only as what it was for: the + macOS case where the copy's filesystem hands the provider a canonically + equivalent spelling of the single name it was given. So a manifest key that + matches an input exactly is that input's record, and the normalized fallback + answers only when exactly one key folds onto it and no other tracked name + shares its normal form (``_census_normalization_collisions``, + ``_manifest_keys_for``). Both ambiguities are counted and stay partial + rather than resolved by position, because picking the first or the last + colliding row is picking which failure to not report. + + Blank rows are the cases the pin's rule makes blank: an extractor error or + an anomalous zero-node extract. They stay partial here. The clean-room + repeat that exited zero in 1.63 s requeued 54 entries; the retained + evidence for that run carries *stamped* rows, so requeueing there is not + observable as a blank row and nothing in this adapter claims it is. That + remains an observed limitation of the incremental gate rather than a shape + this module reports on. + + Nothing upgrades a run: the exit status, a non-empty graph, and the raw + extraction's ``extracted_sources`` are all statements about what was + *dispatched*, failures included, so none of them is success evidence. + Missing evidence, an unparseable manifest, a shape this adapter does not + recognize, an input it could not classify, and a row that cannot be told + apart from a failure all stay ``partial``, which ``graph_status`` refuses + by default. The provider owns no provenance (the evaluation records this as + the first product constraint), so that refusal is the failure an operator + can act on; silently calling it complete is the one they cannot. + """ + if census is None: + return IndexResult( + completeness=PARTIAL, + notes=("build recorded no census to check the provider's manifest against",), + ) + if manifest is None: + return IndexResult( + completeness=PARTIAL, + notes=("provider left no readable manifest of what it processed",), + ) + if digests is None: + return IndexResult( + completeness=PARTIAL, + notes=("build recorded no input digests to check the provider's manifest against",), + ) + if inputs is None: + inputs = _provider_inputs(census) + eligible = inputs.dispatched + rows: dict[str, Any] = {} + folded_keys: dict[str, list[str]] = {} + malformed_rows = 0 + for key, row in manifest.items(): + if not isinstance(key, str): + malformed_rows += 1 + continue + if not isinstance(row, Mapping) or any( + field not in row for field in _MANIFEST_ROW_FIELDS + ): + malformed_rows += 1 + continue + rows[key] = row + folded_keys.setdefault(unicodedata.normalize("NFC", key), []).append(key) + collisions = _census_normalization_collisions(census) + missing = 0 + unstamped = 0 + unreadable = 0 + mismatched = 0 + processed = 0 + unsupported = 0 + collided = 0 + ambiguous = 0 + for path in eligible: + if unicodedata.normalize("NFC", path) in collisions: + # Two tracked names this build cannot tell apart by name, and the + # manifest is keyed by name. Whichever row is found, one successful + # input would be standing in as proof for the other, so neither is + # counted and the run stays partial. + collided += 1 + continue + keys = _manifest_keys_for(path, rows, folded_keys) + if len(keys) > 1: + ambiguous += 1 + continue + row = rows[keys[0]] if keys else None + if row is None: + missing += 1 + continue + digest = row.get("ast_hash") + if not isinstance(digest, str) or not _MANIFEST_HASH.fullmatch(digest): + unstamped += 1 + continue + expected = digests.get(path) + if expected is None: + # The row is well formed and this build cannot say what it should + # have contained. Counting it would be believing the row on its own + # word, which is the whole thing the comparison exists to stop. + unreadable += 1 + elif digest != expected: + mismatched += 1 + elif path in inputs.unsupported: + # Read, not failed, and deterministically not extractable by this + # pin. Counted on its own line rather than as a file this build + # indexed, which it is not. + unsupported += 1 + else: + processed += 1 notes: list[str] = [] - for counter in _INCOMPLETE_COUNTERS: - value = report.get(counter) - if value is True: - notes.append(f"provider reported {counter}") - elif isinstance(value, int) and not isinstance(value, bool) and value > 0: - notes.append(f"provider reported {value} {counter}") - claim = _completion_claim(report) - if claim is False: - notes.append("provider did not report the extraction as complete") - elif claim is None: - notes.append("provider report carried no completion claim") - indexed = _indexed_count(report) - if indexed is None: - notes.append("provider report did not say how many files it indexed") + if malformed_rows: + notes.append(f"provider manifest carried {malformed_rows} unreadable records") + if missing: + notes.append(f"provider manifest does not account for {missing} code files") + if unstamped: + notes.append(f"provider left {unstamped} code files unprocessed or requeued") + if unreadable: + notes.append(f"build could not re-read {unreadable} code files to check their hashes") + if mismatched: + notes.append(f"provider hashed {mismatched} code files that are not the bytes it was given") + if collided: + notes.append( + f"the census carried {collided} code files whose tracked names differ from " + "another tracked name only by Unicode normalization" + ) + if ambiguous: + notes.append( + f"provider manifest carried more than one candidate record for {ambiguous} code files" + ) + if inputs.unclassified: + notes.append( + f"build could not classify {len(inputs.unclassified)} tracked inputs " + "against this provider's own dispatch" + ) + if not eligible: + notes.append("the census carried no code files this provider would index") if notes: - return IndexResult(completeness=PARTIAL, indexed_files=indexed or 0, notes=tuple(notes)) - return IndexResult(completeness=COMPLETE, indexed_files=indexed) + return IndexResult( + completeness=PARTIAL, + indexed_files=processed, + unsupported_inputs=unsupported, + notes=tuple(notes), + ) + return IndexResult( + completeness=COMPLETE, indexed_files=processed, unsupported_inputs=unsupported + ) class _BoundedBuffer(io.BytesIO): @@ -2302,9 +3429,10 @@ def subprocess_indexer( callable that would run whatever a later request's pin happened to name. The argv is the interface the adopt decision evaluated, not a guess at a - conventional one: ``extract`` with the required restrictions and then the - pinned options, in the materialized copy. Everything the provider leaves - behind is then collected and classified from its own report. + conventional one: ``extract``, the scan target the pinned CLI requires, the + required restrictions and then the pinned options, in the materialized copy. + Everything the provider leaves behind is then collected and classified from + its own report. """ command = _resolved_executable(executable) if containment_mechanism() is None: @@ -2335,6 +3463,19 @@ def run(request: IndexRequest) -> IndexResult: "against; no generation was published" ) _refuse_pre_existing_provider_state(request.source_root) + # Before the launch, and only here. These are the bytes this build hands + # the provider; once the child has run, the same tree also holds the + # provider's own output, and a digest taken then would be checking the + # provider's manifest against the provider's own leavings. + # Classified and digested from the copy *before* the launch: after the + # run the same tree also holds provider output, and a shebang read then + # would be classifying whatever the provider left behind. + inputs = ( + _provider_inputs(request.census, request.source_root) + if request.census is not None + else None + ) + digests = _materialized_digests(request.source_root, request.census, inputs) # Built per run, because the boundary is a function of what this build # exposes: the materialized copy and the build's own scratch areas are # writable, the pinned provider's install is readable, and nothing else @@ -2351,7 +3492,7 @@ def run(request: IndexRequest) -> IndexResult: options = _extraction_options(request.pin.options) try: returncode = _run_contained( - [*sandbox, command, _PROVIDER_EXTRACT, *options], + [*sandbox, command, _PROVIDER_EXTRACT, _PROVIDER_SCAN_TARGET, *options], environment=request.environment, cwd=str(request.source_root), timeout=EXTRACTION_TIMEOUT_SECONDS, @@ -2366,9 +3507,11 @@ def run(request: IndexRequest) -> IndexResult: raise ContextError("local graph provider could not be run from its pinned install") from None if returncode != 0: raise ContextError("local graph provider failed; no generation was published") - state_directory = _provider_state_directory(request.source_root) - result = _read_completeness(_provider_report(state_directory)) - _write_private_file(request.output_path, _pack_state(state_directory)) + output_directory = _provider_output_directory(request.source_root) + result = _read_completeness( + _provider_manifest(output_directory), request.census, digests, inputs + ) + _write_private_file(request.output_path, _pack_state(output_directory)) return result return run @@ -2399,6 +3542,13 @@ class BuildManifest: completeness: str skipped_paths: int indexed_files: int + #: Inputs the provider classified as code and deterministically could not + #: extract. Deliberately *not* folded into ``skipped_paths``, which counts + #: tracked entries this build declined to materialize at all (symlinks, + #: submodules, private state). Those two numbers answer different questions + #: -- what this build withheld, and what the provider could not read -- + #: and an operator who needs to act on one cannot act on their sum. + unsupported_inputs: int = 0 def to_json(self) -> dict[str, Any]: return { @@ -2416,6 +3566,7 @@ def to_json(self) -> dict[str, Any]: "completeness": self.completeness, "skipped_paths": self.skipped_paths, "indexed_files": self.indexed_files, + "unsupported_inputs": self.unsupported_inputs, } def shareable_summary(self) -> dict[str, Any]: @@ -2432,11 +3583,21 @@ def shareable_summary(self) -> dict[str, Any]: "graph_digest": self.graph_digest, "graph_bytes": self.graph_bytes, "completeness": self.completeness, + "unsupported_inputs": self.unsupported_inputs, } def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: - """Validate a manifest. Every unreadable shape is a refusal, not a default.""" + """Validate a manifest. Every unreadable shape is a refusal, not a default. + + ``unsupported_inputs`` is accepted as optional so a generation published + before it existed still loads. It is always written, so the only manifests + that take the default are older ones, and ``0`` is the honest reading of + them: that build never counted the provider's unsupported dispatch, and a + zero says the same thing a missing key does. Nothing else is optional -- + an unrecognized key is still a refusal, so this widens what loads by + exactly one name. + """ if not isinstance(payload, Mapping): raise ContextError("local graph manifest must be an object") expected = { @@ -2444,7 +3605,9 @@ def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: "tracked_files", "tracked_bytes", "census_digest", "graph_digest", "graph_bytes", "completeness", "skipped_paths", "indexed_files", } - if set(payload) != expected: + optional = {"unsupported_inputs"} + present = set(payload) + if not expected <= present or not present <= (expected | optional): raise ContextError("local graph manifest fields are missing or unrecognized") if payload["schema"] != MANIFEST_SCHEMA: raise ContextError("unsupported local graph manifest schema") @@ -2474,6 +3637,9 @@ def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: completeness=completeness, skipped_paths=_size(payload["skipped_paths"], MAX_SKIPPED_PATHS), indexed_files=_size(payload["indexed_files"], MAX_TRACKED_FILES), + unsupported_inputs=_size( + payload.get("unsupported_inputs", 0), MAX_TRACKED_FILES + ), ) @@ -3229,6 +4395,9 @@ def build_graph( # at, so what the provider is told to use and what it is # allowed to write are one decision rather than two. writable=(home, temporary), + # The very census those bytes were written from, so the + # indexer measures coverage against what it was given. + census=census, ) ) if not isinstance(result, IndexResult) or result.completeness not in (COMPLETE, PARTIAL): @@ -3257,6 +4426,9 @@ def build_graph( completeness=result.completeness, skipped_paths=len(census.skipped), indexed_files=min(_size(result.indexed_files, MAX_TRACKED_FILES), census.file_count), + unsupported_inputs=min( + _size(result.unsupported_inputs, MAX_TRACKED_FILES), census.file_count + ), ) published = state.publish(manifest, artifact) if not keep_previous: @@ -3473,6 +4645,8 @@ def render_status_text(status: GenerationStatus) -> str: f" census: {manifest.census_digest}", f" graph: {manifest.graph_digest} ({manifest.graph_bytes} bytes)", f" complete: {manifest.completeness}", + f" indexed: {manifest.indexed_files} files " + f"({manifest.unsupported_inputs} unsupported by this provider)", ] ) return "\n".join(lines) + "\n" diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py new file mode 100644 index 00000000..ab44a566 --- /dev/null +++ b/src/code_mower/context_graph_query.py @@ -0,0 +1,1545 @@ +"""Bounded local-graph queries and revision-bound context packets (issue #914). + +``context_graph_lifecycle`` (#913) publishes an immutable generation: an +artifact bound to one commit and tree, with a manifest that says which provider +release produced it and whether that run finished. ``context_graph`` (#876) +scores a delivered packet's citations. Between those two there was nothing: no +way to ask the graph a question and no way to turn an answer into evidence a +recipient may read. + +This module is that middle. It reads the pinned graph schema out of the +published artifact directly rather than through provider query tooling, runs +deterministic bounded traversals for the four questions a code graph answers +better than ordinary repository tools, validates every citation against the +immutable tracked tree of the bound commit, and emits an ordinary +``code_mower.contextPacket.v1`` repository-kind packet. The packet is the whole +interface to a recipient: Claude, Codex and Devin read the same approved bytes +through the existing delivery path, with no graph, no provider install, and no +credentials of their own. + +Two properties the adoption record (``docs/graphify-evaluation.md``) asks for +are load-bearing here and are therefore not configurable: + +* **Queries are symbol-first, relationship-filtered and budgeted.** Default + traversals in the evaluated provider returned 700-900 nodes and truncated + silently. Every traversal here starts from named seeds, follows one filtered + relationship set, and stops at an explicit budget or depth that is reported + as truncation rather than presented as a complete answer -- including when + what stopped it was the requested depth and the node it stopped on still had + eligible relationships behind it. +* **Stale or unknown graph state is never answered from.** A required context + request blocks; an optional one degrades to ordinary repository tools. There + is no third outcome where a consumer is handed an older graph that looks + fresh. + +Nothing here installs, downloads, or executes a provider. The only subprocess +is Git, reading blobs of the commit the generation is already bound to. +""" + +from __future__ import annotations + +import json +import subprocess +import tarfile +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from . import context_graph_lifecycle as lifecycle +from .context_contract import ( + CAPABILITY_VERSION, + PACKET_SCHEMA, + ContextError, + _text, + _timestamp, + normalize_policy, + validate_connection, +) +from .context_graph import MAX_GRAPH_CITATIONS, parse_graph_citation + +#: The graph document Code Mower reads. This is the pinned provider's own +#: ``graph.json`` at commit ``23f2ffa`` (release 0.9.58), which is the release +#: the lifecycle (#913) pins and archives. There is no Code Mower graph schema +#: and no normalization step between the two: an adapter that required a shape +#: the build never produces would reject every real generation, so this module +#: reads the provider's actual output and does the narrowing itself. +GRAPH_MEMBER = "graph.json" + +QUERY_SCHEMA = "code_mower.contextGraphQuery.v1" + +#: The two documents that can appear under ``graph.json``, which are *not* the +#: same file in two dialects and are not read as though they were. +#: +#: ``raw_extraction`` is what the lifecycle's own pinned invocation writes. +#: ``context_graph_lifecycle`` requires ``extract --code-only --no-cluster`` +#: (``_REQUIRED_EXTRACT_OPTIONS``), and the pinned CLI's ``--no-cluster`` branch +#: dumps the merged extractor result directly -- ``nodes``, ``edges``, +#: ``hyperedges``, token counts, ``extracted_sources`` -- through +#: ``write_json_atomic``. It never builds a NetworkX graph, never calls +#: ``to_json``, and therefore writes no ``directed``, ``multigraph``, ``graph`` +#: or ``built_at_commit`` key. Its ``source``/``target`` are the endpoints the +#: extractor's own ``add_edge`` recorded at the call site, so the orientation is +#: the provider's semantic claim and is preserved by reading the edge record. +#: +#: ``node_link`` is ``graphify/export.py::to_json``: a NetworkX +#: ``node_link_data`` document, which always carries ``directed``, +#: ``multigraph``, ``graph`` and ``links``. Its direction is *not* self-evident +#: and is handled separately, below. +GRAPH_FORMAT_RAW = "raw_extraction" +GRAPH_FORMAT_NODE_LINK = "node_link" + +#: Top-level keys only a NetworkX ``node_link_data`` document carries. Any one +#: of them means the file came through ``to_json`` rather than the raw +#: ``--no-cluster`` dump, and it is then read under the node-link rules -- +#: including the direction requirement -- whichever key names its edge list. +NODE_LINK_MARKERS = ("links", "directed", "multigraph", "graph") + +#: The edge list, per format. The raw dump writes ``edges``. ``node_link_data`` +#: writes ``links`` at the pinned commit (``to_json`` passes ``edges="links"``) +#: but NetworkX renamed the key, and the pinned validator accepts either, so a +#: node-link document is read under both names. +RAW_EDGE_KEY = "edges" +NODE_LINK_EDGE_KEYS = ("links", "edges") + +#: Whether a **node-link** export preserves the direction of its relationships, +#: which every question here depends on and no question here can recover. +#: +#: The clustered build writes a NetworkX graph that is undirected by default +#: (``build.py::build_from_json(directed=False)``). Undirected storage +#: canonicalizes endpoint order, so ``source``/``target`` in an undirected +#: export are an endpoint *pair*, not a caller and a callee. The exporter does +#: try to repair that -- it stashes the true endpoints in ``_src``/``_tgt`` and +#: restores them before writing -- but the restored link carries no record that +#: the repair happened, so a reader cannot tell a restored edge from one whose +#: order came out of node iteration, and an undirected build additionally +#: collapses a pair related in both directions onto whichever it saw first. +#: +#: Every answer this module produces is an oriented claim: ``impact`` and +#: ``dependency`` are the same relationships walked in opposite directions, and +#: even a ``symbol`` neighbourhood states "A calls B" rather than "A and B are +#: adjacent". Reading orientation out of a *node-link* document that does not +#: establish it is how a packet comes to assert the reverse of what the code +#: does, so such a document is refused rather than answered from. +#: ``directed: true`` is the provider's own statement that the graph was stored +#: as a ``DiGraph``, where source and target *are* the edge. +#: +#: This check belongs to the node-link format alone. Demanding the marker of a +#: raw extraction would refuse every generation the lifecycle's own pinned +#: options actually produce, since that path emits no marker and has no +#: undirected container to have lost direction in. +GRAPH_DIRECTED_KEY = "directed" + +#: Required node and edge fields, taken from the pinned validator's +#: ``REQUIRED_NODE_FIELDS`` and ``REQUIRED_EDGE_FIELDS``. A record missing one +#: of these is a refusal: the provider's own validator would not have passed +#: it, so a graph carrying it was not produced by the build this adapter was +#: reviewed against. +GRAPH_NODE_FIELDS = ("id", "label", "file_type", "source_file") +GRAPH_EDGE_FIELDS = ("source", "target", "relation", "confidence", "source_file") + +#: The pinned validator's ``VALID_FILE_TYPES``. Only ``code`` is traversable +#: here -- the others are the provider's document/paper/image/rationale/concept +#: corpora, which are not the repository relationships #914 is about and carry +#: no line-checkable location in the bound commit. +GRAPH_FILE_TYPES = frozenset({"code", "document", "paper", "image", "rationale", "concept"}) +CODE_FILE_TYPE = "code" + +#: The pinned validator's ``VALID_CONFIDENCES``, lowercased. The provider +#: writes these uppercase; the packet contract's vocabulary is lowercase, and +#: this is the whole of the difference. +GRAPH_CONFIDENCES = {"EXTRACTED": "extracted", "INFERRED": "inferred", "AMBIGUOUS": "ambiguous"} + +#: A node's or an edge's location in the pinned export is ``source_location``, +#: a string of the form ``L`` written by the extractor's +#: ``add_node``/``add_edge``. Cross-file stubs carry ``""`` -- a real node with +#: no location, which this module keeps traversable and refuses to cite -- and +#: an edge's own ``source_location`` is optional the same way: a call site with +#: no recorded line stays traversable and uncitable rather than refused. +_LOCATION_MAX_LINE = 10_000_000 + +#: The pinned extractor's relation vocabulary, normalized onto the +#: relationships a query filters by. ``contains`` is the extractor's +#: file-to-definition and class-to-member edge, which is what ``defines`` +#: means here; ``implements`` is inheritance by another name. The provider's +#: own word is kept on the edge and is what a packet sentence states -- this +#: mapping only decides which traversals an edge participates in. +#: +#: Relations outside this table are *not* a refusal. The pinned validator does +#: not constrain ``relation`` at all, and the provider's LLM extraction emits +#: relations beyond the extractor's fixed set. An unmapped relation is grouped +#: as ``related``: it is carried, it is citable, and it is reachable by the +#: ``symbol`` neighbourhood, but it never stands in for a ``calls`` or an +#: ``imports`` claim it was not. +GRAPH_RELATIONS = { + "calls": "calls", + "imports": "imports", + "defines": "defines", + "contains": "defines", + "references": "references", + "inherits": "inherits", + "implements": "inherits", + "tests": "tests", +} +OTHER_RELATION = "related" +RELATION_KINDS = frozenset({*GRAPH_RELATIONS.values(), OTHER_RELATION}) + +#: The questions a code graph answers better than ``rg`` and an ordinary +#: reading of the tree, from the comparison set in the adoption record. Each +#: names one direction and one relationship filter; there is no free-form +#: traversal, because a traversal whose shape comes from the question text +#: cannot be bounded or reproduced. +QUESTIONS = ("impact", "dependency", "symbol", "related_tests") + +#: The node kinds a *query* is expressed in. The pinned export does not carry +#: them: a Graphify node declares ``file_type`` (its corpus) and, for a +#: handful of constructs, ``type`` (e.g. ``namespace``) -- never whether it is +#: a file, a definition, or a test. So these are derived, in ``_node_kind``, +#: from the shape the pinned extractor actually emits, and the derivation is +#: named here rather than hidden because it is the one place this adapter +#: infers something the provider did not say. +NODE_KINDS = frozenset({"file", "symbol", "test"}) + +#: How the provider's own qualification of a relationship maps onto the packet +#: contract's confidence vocabulary. ``ambiguous`` is the important one: the +#: provider resolved the relationship to more than one candidate, which is a +#: claim a recipient must be able to see as unresolved rather than read as +#: fact. The contract has no ``ambiguous`` value, so it maps to ``unknown`` +#: and also raises the ``unresolved_entities`` omission on the packet. +EVIDENCE_CONFIDENCE = {"extracted": "extracted", "inferred": "inferred", "ambiguous": "unknown"} + +#: Path segments that make a code file a test in this repository's own layout. +#: A derivation, like ``_node_kind`` itself, and deliberately conservative: +#: naming a non-test file a test would put it in a ``related_tests`` answer. +_TEST_PREFIXES = ("tests/", "test/") +_TEST_DIRECTORIES = ("/tests/", "/test/") +_TEST_STEM_SUFFIXES = ("_test", "_spec") + +#: Relationship filters per question, and whether the traversal runs along +#: edges or against them. ``impact`` asks who is affected by a change, which is +#: the reverse of ``dependency`` over the same relationships. The filters are +#: written in the normalized vocabulary of ``GRAPH_RELATIONS``, so an +#: ``implements`` edge participates wherever ``inherits`` does and an unmapped +#: relation participates only in the ``symbol`` neighbourhood. +_TRAVERSALS: dict[str, tuple[str, frozenset[str]]] = { + "impact": ("incoming", frozenset({"calls", "imports", "references", "tests", "inherits"})), + "dependency": ("outgoing", frozenset({"calls", "imports", "references", "inherits"})), + "symbol": ("both", RELATION_KINDS), + "related_tests": ("incoming", frozenset({"tests", "calls", "references"})), +} + +#: Depth ceilings per question. ``symbol`` is a neighbourhood, not a walk: +#: what defines this name and what touches it directly. +_DEFAULT_DEPTH = {"impact": 2, "dependency": 2, "symbol": 1, "related_tests": 2} +MAX_DEPTH = 4 + +#: Budgets. The node budget is what stops a traversal; the rest bound what one +#: packet may carry and are enforced before the contract's own limits so that +#: an over-budget answer is reported as truncated rather than refused at +#: delivery. +DEFAULT_NODE_BUDGET = 40 +MAX_NODE_BUDGET = 200 +#: The adapter's own hard ceiling on documents per packet. The selected +#: policy's ``max_documents`` applies on top of it and only ever downward: a +#: packet carries the smaller of the two, and a policy asking for more than this +#: still gets this. +MAX_DOCUMENTS = 16 +MAX_CITATIONS_PER_DOCUMENT = 10 +MAX_SEEDS = 8 + +#: Graph bounds. The artifact is already bounded to ``MAX_ARTIFACT_BYTES`` by +#: the lifecycle and its digest is verified before this module reads it, so +#: these bound what one *member* may cost this process to hold and parse. +MAX_GRAPH_BYTES = 64 * 1024 * 1024 +MAX_NODES = 200_000 +MAX_EDGES = 500_000 + +#: How much of a blob to read while confirming one line claim. A claim is +#: confirmed as soon as its last claimed line is seen, so a citation into a +#: large generated file costs the lines up to the claim. +_BLOB_CHUNK_BYTES = 256 * 1024 + +#: The syntactic argument list the pinned extractor appends to a callable's +#: label. A real 0.9.58 graph of this repository labels the function +#: ``parse_graph_citation`` as ``parse_graph_citation()`` and the function +#: ``packet`` as ``packet()``, so the label a human would type and the label the +#: provider wrote differ by exactly this suffix. +_CALLABLE_OPEN = "(" +_CALLABLE_CLOSE = ")" + + +def _callable_base(label: str) -> str: + """A callable label's name with only its trailing argument list removed. + + ``""`` for anything that is not a callable label, and that emptiness is + load-bearing: ``seed_matches`` compares this against a target that ``_text`` + has already rejected as empty, so a non-callable label can never match by + both sides being blank. + + Deliberately syntactic and deliberately narrow. The suffix must begin at the + label's *first* ``(``, must be a balanced parenthesized group, and that group + must close on the label's last character; the name before it must be + non-empty and carry no ``)`` of its own. So ``render(int)``, + ``render(Callable[(int)])`` and ``render()`` all reduce to ``render`` while + ``render(int))``, ``render()x``, ``render(int`` and ``()`` reduce to nothing. + It is not a parser: the pinned extractor writes the label, this reads the one + suffix it writes, and everything else stays an exact comparison. + """ + if not label.endswith(_CALLABLE_CLOSE): + return "" + opened = label.find(_CALLABLE_OPEN) + if opened <= 0: + return "" + base = label[:opened] + if _CALLABLE_CLOSE in base: + return "" + depth = 0 + for index in range(opened, len(label)): + character = label[index] + if character == _CALLABLE_OPEN: + depth += 1 + elif character == _CALLABLE_CLOSE: + depth -= 1 + if depth == 0: + # The first group closes here. Anything after it means the tail + # is not one argument list, so the label is left alone. + return base if index == len(label) - 1 else "" + return "" + + +def _location_citation(path: str, line: int | None) -> str | None: + """A path and an optional line as a citation, or ``None`` with no path. + + Shared by ``GraphNode`` and ``GraphEdge``: the pinned export records a + single line per node or call site, not a span, so a located citation + states one line. Claiming a span the provider never stated would be this + adapter inventing the extent of a definition or a call. + """ + if not path: + return None + if line is None: + return path + return f"{path}#L{line}" + + +@dataclass(frozen=True) +class GraphNode: + """One node of the pinned export, narrowed and held to the citation rules. + + ``path`` is the export's ``source_file`` and ``line`` is its + ``source_location``. Both may be absent: the pinned extractor emits + *sourceless stubs* for cross-file references it could not resolve locally + (``source_file`` and ``source_location`` set to ``""``), so that a + corpus-level pass can collapse them onto a real definition. Those nodes are + real relationships and stay traversable; they are simply not citable, and + ``citation`` is ``None`` for them rather than a path that points nowhere. + """ + + id: str + kind: str + name: str + path: str + line: int | None + + @property + def citation(self) -> str | None: + """The node's location as a citation, or ``None`` if it has no location.""" + return _location_citation(self.path, self.line) + + +@dataclass(frozen=True) +class GraphEdge: + """One relationship, with the provider's own word for it and its own caveat. + + ``relation`` is what the provider wrote; ``kind`` is that relation + normalized onto ``GRAPH_RELATIONS`` for filtering. A packet sentence states + ``relation``, so a recipient reads the provider's claim and not this + module's grouping of it. + + ``path`` and ``line`` are the provider's own ``source_file`` and + ``source_location`` for this call site, held to the same citation rules as + a node's. Two edges that agree on every other field but were written at + different call sites are two distinct relationships, not one repeated: + without the call site, ``run_query`` had no way to tell "the same claim, + twice" from "the same claim, from two different places in the code" and + silently kept only one. + """ + + source: str + target: str + relation: str + kind: str + evidence: str + path: str + line: int | None + + @property + def citation(self) -> str | None: + """The call site's location as a citation, or ``None`` if it has none.""" + return _location_citation(self.path, self.line) + + +#: Sorts below every real provider line, which is always ``>= 1``, so a call +#: site with no line still has a total order against one that has it -- and +#: does so without giving ``line`` a type that mixes ``None`` into a sort key. +_NO_LINE = -1 + + +def _edge_identity(edge: GraphEdge) -> tuple[str, str, str, str, str, str, int]: + """The one identity a retained edge is deduplicated and ordered by. + + Source, target, the provider's own relation word, the normalized kind, + the normalized confidence, and the call site's path and line, together. + Two call-site records that agree on everything else but were written at + different lines -- or in different files -- are two relationships, and + dropping either the path or the line back onto ``None`` is how a reader + would silently collapse them back into one. This is the single identity + ``run_query`` deduplicates and reports truncation against, and the single + key ``load_graph`` and ``_grouped`` sort by, so a permutation of the + provider's node or edge order can never change what a bounded traversal + reports. + """ + return ( + edge.source, edge.target, edge.relation, edge.kind, edge.evidence, + edge.path, edge.line if edge.line is not None else _NO_LINE, + ) + + +@dataclass(frozen=True) +class CodeGraph: + """A parsed, bounded, deterministically ordered graph of one generation.""" + + generation: str + commit: str + nodes: Mapping[str, GraphNode] + edges: tuple[GraphEdge, ...] + outgoing: Mapping[str, tuple[GraphEdge, ...]] + incoming: Mapping[str, tuple[GraphEdge, ...]] + #: Code nodes that lost at least one relationship because its other endpoint + #: is an id the provider's node list never declared at all. Distinct from a + #: relationship onto a corpus this module deliberately does not query: that + #: endpoint *was* declared, and dropping it is a stated scope rather than + #: missing evidence. A traversal that touches one of these nodes is reporting + #: a neighbourhood the document itself could not state in full, and says so. + incomplete: frozenset[str] = frozenset() + + def seed_matches(self, target: str) -> tuple[tuple[GraphNode, ...], bool]: + """The seeds a target names, and whether the seed bound dropped any. + + Symbol-first, as the adoption record requires: a bare name resolves to + the symbols that carry it, and only a target that names no symbol at + all is read as a path. Ordered by id so two runs against one generation + seed identically. + + Resolution is three ordered tiers, and a later tier is consulted only + when every earlier one is empty, so a literal label always wins over + the same string read as a bare callable and both win over a path: + + 1. The provider's label, exactly as written. + 2. The provider's *canonical callable label* with only its syntactic + trailing argument list removed (``_callable_base``). A real pinned + graph labels a function ``parse_graph_citation()``, so without this + tier the natural bare spelling of a function name resolves to + nothing and every query about it answers "unresolved". + 3. The path, exactly as written. + + Tier 2 is an equality test against a label with one suffix stripped -- + not a prefix, substring, or edit-distance match. ``parse_graph`` does + not reach ``parse_graph_citation()``, and two overloads that differ + only in their argument lists both match, which is real ambiguity and is + reported as such rather than resolved by picking one. + + The overflow flag is not cosmetic. A name carried by more than + ``MAX_SEEDS`` definitions has definitions this traversal will never + start from, and every relationship reachable only from those is absent + from the answer. Silently slicing here would let ``run_query`` report a + complete, untruncated result over a graph it only partly read, which is + exactly the failure the adoption record's condition 4 is about. + """ + name = _text(target, maximum=512) + matches = [node for node in self.nodes.values() if node.name == name] + if not matches: + matches = [ + node for node in self.nodes.values() if _callable_base(node.name) == name + ] + if not matches: + matches = [node for node in self.nodes.values() if node.path == name] + ordered = tuple(sorted(matches, key=lambda node: node.id)) + return ordered[:MAX_SEEDS], len(ordered) > MAX_SEEDS + + def seeds(self, target: str) -> tuple[GraphNode, ...]: + """The bounded seed set alone, for callers that do not report truncation.""" + return self.seed_matches(target)[0] + + +def _required(value: Any, fields: Sequence[str], *, what: str) -> Mapping[str, Any]: + """A provider record with every field its own validator requires. + + Required fields only. Unrecognized *extra* keys are not a refusal here, + which is a deliberate change from reading an invented schema: the pinned + exporter already annotates nodes with ``community``, ``community_name`` and + ``norm_label``, edges with ``confidence_score`` and ``weight``, and either + with a free-form ``metadata`` dict whose contents come from an LLM + extraction. None of those change a traversal. Refusing them would reject + every real generation -- which is exactly what a strict reader of a + hand-written schema did. + + What *is* strict: every field this module reads is read by name, bounded, + and validated against the pinned vocabulary. Nothing else is looked at. + """ + if not isinstance(value, Mapping): + raise ContextError(f"local graph {what} must be an object") + missing = [field_name for field_name in fields if field_name not in value] + if missing: + raise ContextError(f"local graph {what} is missing required provider fields") + return value + + +def _maybe_text(value: Any, *, maximum: int) -> str: + """Bounded single-line text, or ``""`` for a field the provider left empty. + + ``_text`` rejects the empty string, which is correct for every identifier + in the packet contract and wrong for exactly one field here: a sourceless + stub's ``source_file``. Everything non-empty goes through ``_text`` + unchanged, so the bound and the control-character rules are the same ones. + """ + if value is None or value == "": + return "" + return _text(value, maximum=maximum) + + +def _location(value: Any) -> int | None: + """Parse the pinned export's ``source_location``: ``L``, or nothing. + + The extractor writes ``f"L{line}"``; a sourceless stub writes ``""``. A + value in any other shape is a refusal rather than a node with an unknown + location, because a location this module cannot read is one it cannot + check against the bound commit. + """ + if value is None or value == "": + return None + text = _text(value, maximum=32) + suffix = text[1:] + # ``str.isdigit()`` accepts Unicode digits ``int()`` cannot parse (a + # superscript ``\u00b2`` is a digit and not a decimal), which would let a + # value like ``"L\u00b2"`` escape this refusal as an unhandled + # ``ValueError`` instead of the bounded ``ContextError`` every other + # unsupported location shape raises. Requiring ASCII first closes that. + if not text.startswith("L") or not suffix.isascii() or not suffix.isdigit(): + raise ContextError("local graph source location is not a supported provider location") + line = int(suffix) + if not 1 <= line <= _LOCATION_MAX_LINE: + raise ContextError("local graph line number is out of range") + return line + + +def _node_kind(path: str, label: str, node_type: Any) -> str: + """Derive a query-level node kind from what the pinned export does carry. + + The provider states no such kind, so this reads the shape its extractor + emits: + + * A **file** node is the one the extractor creates per file, whose label is + that file's base name at ``L1`` (``add_node(_make_id(str(path)), + path.name, 1)``). Matching on the base name is what distinguishes it from + a definition inside the same file. + * A **test** is a code node whose file sits in this repository's test + layout. A path convention, not a provider claim -- ``related_tests`` + answers from it, so it is kept narrow. + * Everything else is a **symbol**: a definition, a member, a namespace, or + an unresolved cross-file stub. + """ + if node_type == "namespace": + return "symbol" + if path: + base = path.rsplit("/", 1)[-1] + lowered = path.lower() + stem = base.lower().rsplit(".", 1)[0] + is_test = ( + lowered.startswith(_TEST_PREFIXES) + or any(directory in lowered for directory in _TEST_DIRECTORIES) + or stem.startswith("test_") + or stem.endswith(_TEST_STEM_SUFFIXES) + ) + if is_test: + return "test" + if base == label: + return "file" + return "symbol" + + +@dataclass(frozen=True) +class _ParsedNode: + """One provider node record: its declared id, and the code node it is or is not. + + The id survives the ``code``-only filter on purpose. An edge onto an id the + document *declared* as a document, paper, image, rationale or concept is a + relationship this module has stated it does not query; an edge onto an id + the document never declared at all is evidence the provider itself could + not state. Without keeping the declared ids those two are the same dangling + edge, and ``_edge`` cannot tell a scope decision from missing evidence. + """ + + id: str + node: GraphNode | None + + +def _node(value: Any) -> _ParsedNode: + """One pinned-export node: a queryable code node, or a declared exclusion. + + A non-``code`` node is dropped rather than refused. The provider indexes + documents, papers, images, rationales and concepts into the same graph, and + those are not repository relationships: they carry no location in the bound + commit, so no traversal here could cite one. Dropping them is bounded -- + every edge that named one becomes a dangling edge, which ``load_graph`` + prunes. No count of the dropped records is kept or reported: what a packet + states about its own incompleteness is the traversal's truncation and + omission fields, not a tally of corpora this module never queries. Their + *ids* are kept, and only so that ``_edge`` can tell this stated exclusion + apart from an endpoint the document never carried. + """ + record = _required(value, GRAPH_NODE_FIELDS, what="node") + # Bounded text before membership: a vocabulary field is looked up in a set, + # and a JSON array or object there is unhashable, so testing it first + # raises TypeError out of a reader whose only failure is ``ContextError``. + file_type = _text(record["file_type"], maximum=64) + if file_type not in GRAPH_FILE_TYPES: + raise ContextError("unsupported local graph node file type") + identifier = _text(record["id"], maximum=512) + if file_type != CODE_FILE_TYPE: + return _ParsedNode(id=identifier, node=None) + path = _maybe_text(record["source_file"], maximum=1024) + label = _text(record["label"], maximum=512) + node = GraphNode( + id=identifier, + kind=_node_kind(path, label, record.get("type")), + name=label, + path=path, + line=_location(record.get("source_location")), + ) + # Held to the citation rules here, at parse time, rather than when a packet + # is written: a node that could never be cited inside the indexed checkout + # must not be traversable either, or an out-of-scope path reaches a + # recipient as a relationship whose citation was quietly dropped. A + # sourceless stub has nothing to hold to the rules and is exempt. + if node.citation is not None: + parse_graph_citation(node.citation) + return _ParsedNode(id=identifier, node=node) + + +@dataclass(frozen=True) +class _ParsedEdge: + """One provider link: the relationship it is, or the evidence it costs. + + ``incomplete`` names the endpoints that *are* in the graph on a link whose + other end the document never declared. Those are the nodes whose reported + neighbourhood is smaller than the provider's own, so a traversal reaching + one of them has to say the answer is partial rather than call it complete. + Both fields empty is the third case, and the only silent one: a link whose + missing endpoints were all declared exclusions. + """ + + edge: GraphEdge | None + incomplete: tuple[str, ...] + + +def _edge( + value: Any, nodes: Mapping[str, GraphNode], excluded: frozenset[str] +) -> _ParsedEdge: + """One pinned-export link, and what dropping it costs when it is dropped. + + Pruned rather than refused, which is the pinned exporter's own treatment: + ``export.py::prune_dangling_edges`` drops links whose endpoints are not in + the node set and reports a count. But the two links that reach that path + are not the same fact and must not be reported as one: + + * A link onto a node ``_node`` dropped for its corpus. The document + declared that endpoint and this module has stated it does not query that + corpus, so the relationship is out of scope by a rule a recipient can + read. Dropped silently, as before. + * A link onto an id the node list never carries at all. The provider + emitted a relationship and then did not describe one of its ends, so this + is evidence the document is missing -- not a scope this module chose. + Reporting a complete answer over such a neighbourhood would state that + nothing was left out when something was. The surviving endpoint is named + so the traversal can raise ``provider_partial`` if it reaches it. + + A link whose *every* endpoint is undeclared names no node any traversal can + start from or reach, so there is nothing to attribute it to and nothing to + report: no query's answer can be narrowed by it. + """ + record = _required(value, GRAPH_EDGE_FIELDS, what="edge") + # Bounded text first, for the same reason as a node's ``file_type``: the + # lookup below is a dict membership test, which a non-text value turns into + # a TypeError instead of the refusal this module promises. + confidence = _text(record["confidence"], maximum=64) + if confidence not in GRAPH_CONFIDENCES: + raise ContextError("unsupported local graph edge confidence") + relation = _text(record["relation"], maximum=128) + source = _text(record["source"], maximum=512) + target = _text(record["target"], maximum=512) + endpoints = ((source, source in nodes), (target, target in nodes)) + if all(present for _, present in endpoints): + built = GraphEdge( + source=source, + target=target, + relation=relation, + kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), + evidence=GRAPH_CONFIDENCES[confidence], + path=_maybe_text(record["source_file"], maximum=1024), + line=_location(record.get("source_location")), + ) + # Held to the same citation rules as a node's, at parse time: a call + # site this adapter could never point at inside the indexed checkout + # must not stay traversable as though it were verified evidence. + if built.citation is not None: + parse_graph_citation(built.citation) + return _ParsedEdge(edge=built, incomplete=()) + if all(present or endpoint in excluded for endpoint, present in endpoints): + # Every absent end was a node the document declared and this module + # deliberately does not query. A stated scope, not missing evidence. + return _ParsedEdge(edge=None, incomplete=()) + return _ParsedEdge( + edge=None, + incomplete=tuple(endpoint for endpoint, present in endpoints if present), + ) + + +def _graph_format(payload: Mapping[str, Any]) -> str: + """Which of the provider's two ``graph.json`` documents this is. + + The discriminator is the presence of a NetworkX node-link marker, not the + name of the edge list. ``node_link_data`` always writes ``directed``, + ``multigraph`` and ``graph`` alongside its links, and the raw + ``--no-cluster`` dump writes none of them -- it writes the extractor's own + merged result, whose only structural keys are ``nodes`` and ``edges``. + Keying off ``edges`` instead would misread a newer NetworkX node-link + document, which names its links ``edges``, as a raw extraction and so skip + the direction requirement the node-link format needs. + """ + if any(key in payload for key in NODE_LINK_MARKERS): + return GRAPH_FORMAT_NODE_LINK + return GRAPH_FORMAT_RAW + + +def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdge, ...]]: + """Adjacency in one fixed order, so a traversal cannot depend on input order.""" + buckets: dict[str, list[GraphEdge]] = {} + for edge in edges: + buckets.setdefault(getattr(edge, by), []).append(edge) + return { + key: tuple(sorted(group, key=_edge_identity)) + for key, group in buckets.items() + } + + +def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> CodeGraph: + """Read the pinned provider's ``graph.json`` into a bounded queryable graph. + + The document is the pinned provider's own output, so what is validated here + is the provider's own contract -- the required fields of its validator, its + ``file_type`` and ``confidence`` vocabularies, its ``L`` locations -- + and not a shape Code Mower invented. Every value this module reads is read + by name and bounded; every value it does not read is left alone. + + The *supported* document is the raw extraction the lifecycle's pinned + ``extract --code-only --no-cluster`` writes. A NetworkX node-link export is + also read, because a generation may have been produced by the clustered + path, but its provenance differs and it is held to the extra direction + requirement that path needs. The two are told apart structurally in + ``_graph_format``, never by guessing from an edge key. + + Two provenance checks are worth more than any field check. The first is + ``built_at_commit``: the *exporter* stamps the commit the graph was built + from, and if that disagrees with the commit the generation is bound to then + the artifact and the manifest describe different revisions, which is a + refusal no traversal should be run past. The raw path writes no such stamp + -- it bypasses ``to_json`` entirely -- so for it this check is vacuous and + the binding rests on the lifecycle's own commit binding and on the second + check: the census every citation goes through later. + """ + if not isinstance(payload, Mapping): + raise ContextError("local graph document must be an object") + graph_format = _graph_format(payload) + raw_nodes = payload.get("nodes") + if graph_format == GRAPH_FORMAT_RAW: + raw_edges = payload.get(RAW_EDGE_KEY) + else: + raw_edges = next((payload[key] for key in NODE_LINK_EDGE_KEYS if key in payload), None) + if raw_edges is None or raw_nodes is None: + raise ContextError("local graph document carries no provider nodes and edges") + if not isinstance(raw_nodes, list) or len(raw_nodes) > MAX_NODES: + raise ContextError("local graph node count exceeds its budget") + if not isinstance(raw_edges, list) or len(raw_edges) > MAX_EDGES: + raise ContextError("local graph edge count exceeds its budget") + # Before a single edge is read, because direction is not a property of any + # one link: an undirected node-link export's endpoints are a pair the + # storage ordered, and no traversal, filter or sentence below can be honest + # about a relationship whose orientation the document never stated. A raw + # extraction is exempt because its endpoints never passed through a NetworkX + # container at all -- see ``GRAPH_DIRECTED_KEY``. + if graph_format == GRAPH_FORMAT_NODE_LINK and payload.get(GRAPH_DIRECTED_KEY) is not True: + raise ContextError( + "local graph node-link export does not preserve relationship direction; " + "rebuild the generation with the pinned --no-cluster extraction or as a " + "directed graph" + ) + stamped = payload.get("built_at_commit") + if stamped is not None and _text(stamped, maximum=64) != commit: + raise ContextError("local graph was built from a different commit than its generation") + nodes: dict[str, GraphNode] = {} + excluded: set[str] = set() + for value in raw_nodes: + parsed = _node(value) + node = parsed.node + if node is None: + excluded.add(parsed.id) + continue + if node.id in nodes: + # The provider's own validator does not check this, but neither of + # its write paths can produce it: the raw dump runs its node list + # through ``build.dedupe_nodes`` and the clustered one through a + # NetworkX node set, and both collapse same-id nodes. A document + # that carries two was not written by the pinned provider. + raise ContextError("local graph node identifiers must be unique") + nodes[node.id] = node + frozen = frozenset(excluded) + parsed_edges = [_edge(value, nodes, frozen) for value in raw_edges] + edges = tuple(sorted( + (item.edge for item in parsed_edges if item.edge is not None), + key=_edge_identity, + )) + # Attributed to the surviving endpoint rather than counted: a traversal that + # never reaches one of these nodes is not answering over missing evidence + # and must not claim it is, and one that does reach it has to say so. + incomplete = frozenset( + endpoint for item in parsed_edges for endpoint in item.incomplete + ) + return CodeGraph( + generation=generation, + commit=commit, + nodes=nodes, + edges=edges, + outgoing=_grouped(edges, by="source"), + incoming=_grouped(edges, by="target"), + incomplete=incomplete, + ) + + +def read_graph(state: lifecycle.GraphStateRoot, status: lifecycle.GenerationStatus) -> CodeGraph: + """Read the pinned schema out of a generation whose digest already matched. + + ``status`` must be a usable verdict from ``graph_status``: that is what + bound the artifact to a commit and verified its digest, and re-deriving + either here would be a second answer to a question already settled. + + The member is read from the archive as a stream and bounded as it is read. + The whole artifact is already inside the lifecycle's budget, but one member + of it is not: a compressed or sparse entry can declare a size this process + would otherwise allocate before looking at it. + """ + if not status.usable or status.manifest is None or status.generation is None: + raise ContextError("local graph is not usable for queries") + path = state.artifact_path(status.generation) + try: + with tarfile.open(path, mode="r:") as archive: + try: + info = archive.getmember(GRAPH_MEMBER) + except KeyError: + raise ContextError("local graph generation carries no pinned graph document") from None + if not info.isfile() or info.size > MAX_GRAPH_BYTES: + raise ContextError("local graph document is missing or exceeds its budget") + stream = archive.extractfile(info) + if stream is None: + raise ContextError("local graph document is unreadable") + raw = stream.read(MAX_GRAPH_BYTES + 1) + except (OSError, tarfile.TarError): + raise ContextError("local graph artifact is unreadable") from None + if len(raw) > MAX_GRAPH_BYTES: + raise ContextError("local graph document exceeds its budget") + try: + payload = json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + raise ContextError("local graph document is not supported JSON") from None + return load_graph(payload, generation=status.generation, commit=status.manifest.commit) + + +@dataclass(frozen=True) +class Relation: + """One traversal result: a reached node, the edge that reached it, and both ends. + + ``origin`` is the *other endpoint of ``via``* -- the node the walk expanded + when it found this one -- and never the seed it started from. Those differ + from the second hop onwards, and conflating them is how a traversal comes + to assert a relationship the graph does not carry: a two-hop walk from + ``parse_config`` that reaches ``render`` through ``load`` would otherwise + read as "render calls parse_config" and cite two locations that have no + edge between them. ``seed`` keeps the provenance that conflation was + standing in for, without putting it in the claim. + """ + + node: GraphNode + via: GraphEdge + origin: GraphNode + seed: GraphNode + depth: int + + +@dataclass(frozen=True) +class QueryResult: + """A bounded traversal, with everything a consumer needs to distrust it.""" + + question: str + target: str + generation: str + commit: str + seeds: tuple[GraphNode, ...] + relations: tuple[Relation, ...] + truncated: bool + ambiguous: bool + omissions: tuple[str, ...] + + @property + def resolved(self) -> bool: + return bool(self.seeds) + + def shareable_summary(self) -> dict[str, Any]: + """Metadata only: counts, states and the binding, never indexed content.""" + return { + "schema": QUERY_SCHEMA, + "question": self.question, + "generation": self.generation, + "source_revision": self.commit, + "seeds": len(self.seeds), + "relations": len(self.relations), + "truncated": self.truncated, + "ambiguous": self.ambiguous, + "omissions": list(self.omissions), + } + + +def _neighbours(graph: CodeGraph, node_id: str, direction: str, kinds: frozenset[str]) -> list[tuple[GraphEdge, str]]: + """Filtered adjacency in a fixed order: the only place direction is read.""" + found: list[tuple[GraphEdge, str]] = [] + if direction in ("outgoing", "both"): + found.extend((edge, edge.target) for edge in graph.outgoing.get(node_id, ()) if edge.kind in kinds) + if direction in ("incoming", "both"): + found.extend((edge, edge.source) for edge in graph.incoming.get(node_id, ()) if edge.kind in kinds) + return found + + +def run_query( + graph: CodeGraph, + *, + question: str, + target: str, + depth: int | None = None, + node_budget: int = DEFAULT_NODE_BUDGET, +) -> QueryResult: + """One deterministic, bounded, relationship-filtered traversal. + + Breadth-first from the seeds in a fixed order, so the same generation and + the same question produce the same answer every time and a budget cut + removes the *furthest* relationships rather than arbitrary ones. Reaching + the budget sets ``truncated``; it never silently shortens the answer. + + Node expansion and relationship reporting are bounded separately. A node is + walked through once; a distinct directed relationship is reported once, + including when both of its endpoints have already been seen. Anything left + out is left out by the budget or the depth limit, and says so: a depth + boundary that still has eligible, unreported relationships behind it sets + ``truncated`` exactly as the budget does, and one that has none -- a chain + that ends there, or a cycle whose boundary edges are already in the answer + -- leaves the result complete. + """ + if question not in QUESTIONS: + raise ContextError("unsupported local graph question") + if type(node_budget) is not int or not 1 <= node_budget <= MAX_NODE_BUDGET: + raise ContextError("local graph node budget is out of range") + limit = _DEFAULT_DEPTH[question] if depth is None else depth + if type(limit) is not int or not 1 <= limit <= MAX_DEPTH: + raise ContextError("local graph traversal depth is out of range") + direction, kinds = _TRAVERSALS[question] + seeds, seed_overflow = graph.seed_matches(target) + omissions: list[str] = [] + if not seeds: + return QueryResult( + question=question, target=target, generation=graph.generation, commit=graph.commit, + seeds=(), relations=(), truncated=False, ambiguous=False, + omissions=("unresolved_entities",), + ) + # More than one definition carries the target's name, so every relationship + # below is reported from a seed set the provider could not disambiguate -- + # and a name with more definitions than the seed bound allows is the same + # uncertainty, only worse. + ambiguous = len(seeds) > 1 or seed_overflow + # Two separate identities, because they answer two separate questions. + # ``expanded`` bounds the *walk*: a node is stepped through once, which is + # what keeps the traversal linear and terminating. ``reported`` bounds the + # *answer*: a distinct directed relationship is stated once. Sharing one + # set between them silently deleted evidence -- if A calls B and B calls A, + # the second edge was suppressed because its endpoint had been walked, and + # every relationship among a path's seeds disappeared because all of its + # endpoints were seeds -- while the result still claimed to be complete. + expanded = {node.id for node in seeds} + # Did this traversal read a neighbourhood the provider's own document could + # not state in full? Set from the nodes the walk actually touches, never + # from the graph as a whole: a dangling endpoint somewhere else in the + # repository is not a hole in *this* answer, and marking every query partial + # because of one would make the flag mean nothing. + incomplete = any(node.id in graph.incomplete for node in seeds) + reported: set[tuple[str, str, str, str, str, str, int]] = set() + relations: list[Relation] = [] + over_budget = False + frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] + beyond_depth = False + while frontier: + node, seed, level = frontier.pop(0) + if level >= limit: + # The requested depth stops the walk here, and stopping is allowed. + # Stopping *quietly* is not: for A -> B -> C -> D at depth 2 the + # answer ends at C, and a reader told the answer is complete would + # conclude C depends on nothing. So ask what this node would have + # contributed, and say so if the answer is anything. + # + # "Anything" is measured the same way the walk measures it: an + # eligible relationship under this question's direction and + # relationship filter whose identity is not already reported. That + # is what keeps the flag honest in both directions. A chain that + # genuinely ends at the boundary contributes nothing and stays + # complete; a cycle or a ``symbol`` neighbourhood whose boundary + # edges were already stated from the other side contributes + # nothing either, because those identities are already in the + # answer; and a parallel edge the provider worded differently is a + # different identity, so it counts. Reading adjacency costs no + # budget and reports nothing -- it only decides the flag. + # + # Ordering makes this exact rather than approximate: the frontier + # is FIFO and levels never decrease, so every node below the + # boundary has already been expanded by the time the first + # boundary node is popped. ``reported`` is final here. + if not beyond_depth and any( + _edge_identity(edge) not in reported + for edge, _ in _neighbours(graph, node.id, direction, kinds) + ): + beyond_depth = True + continue + for edge, other_id in _neighbours(graph, node.id, direction, kinds): + # The provider's own record, endpoints, wording and call site + # together: two parallel edges that say different things about the + # same pair -- or say the same thing from two different call sites + # -- are two relationships, a self-loop reached from both sides is + # one, and a byte-identical duplicate record is one. + identity = _edge_identity(edge) + if identity in reported: + continue + if len(relations) >= node_budget: + over_budget = True + break + reported.add(identity) + reached = graph.nodes[other_id] + incomplete = incomplete or other_id in graph.incomplete + # ``node``, not ``seed``: the relationship being reported is the one + # this edge carries, between the node the walk expanded and the node + # it just reached. The seed travels alongside as provenance. + relations.append(Relation(node=reached, via=edge, origin=node, seed=seed, depth=level + 1)) + # Reporting the edge never re-queues an endpoint the walk has + # already stepped through, so retaining these relationships costs + # the bound nothing: the frontier still holds each node once. + if other_id not in expanded: + expanded.add(other_id) + frontier.append((reached, seed, level + 1)) + if over_budget: + break + if question == "related_tests": + # Relationship-filtered is not the same as answer-filtered: the walk + # reaches callers so that a test two hops away is found, but only the + # tests are the answer. + relations = [item for item in relations if item.node.kind == "test"] + # Three different ways to have left something out, reported as one state: a + # relationship budget that stopped the walk, a seed bound that stopped it + # from ever starting at some of the target's definitions, and a depth limit + # that stopped it at a node with relationships nobody asked it to drop + # silently. All three are the same claim to a recipient -- this question has + # more evidence than this answer carries -- so all three raise + # ``provider_has_more`` rather than inventing a fourth code the packet + # contract does not define. ``provider_partial`` stays what it was: evidence + # the provider's document never carried at all, which no bound of ours cut. + truncated = over_budget or seed_overflow or beyond_depth + if truncated: + omissions.append("provider_has_more") + if ambiguous or any(item.via.evidence == "ambiguous" for item in relations): + omissions.append("unresolved_entities") + if incomplete: + # The provider declared a relationship onto an end it never described, + # so the node this walk read has neighbours no reader of this document + # can name. Not ``truncated`` -- no budget and no depth limit cut this, + # the evidence was never in the artifact -- but still partial. + omissions.append("provider_partial") + return QueryResult( + question=question, target=target, generation=graph.generation, commit=graph.commit, + seeds=seeds, relations=tuple(relations), truncated=truncated, ambiguous=ambiguous, + omissions=tuple(dict.fromkeys(omissions)), + ) + + +class CitationValidator: + """Confirm citations against the immutable tracked tree of the bound commit. + + Not against the working tree, which is the point. The generation binds one + commit; the checkout it was built from has since been edited, rebased, or + left dirty, and a line claim confirmed against an edited file is a claim + confirmed against a revision nobody asked about. Every path is checked + against that commit's census -- so an untracked, ignored, or since-deleted + file is never cited -- and every line claim is confirmed against the blob + the census names. + + Blob line counts are memoized per blob, so a packet that cites one file + many times reads it once, and each read stops at the claimed line. + """ + + def __init__(self, repository: Path, census: lifecycle.TrackedCensus): + self._repository = repository + self._blobs = {entry.path: entry for entry in census.entries} + self._counts: dict[str, int | None] = {} + + def tracked(self, path: str) -> bool: + return path in self._blobs + + def validate(self, citation: str) -> bool: + """True when the path is tracked at the bound commit and the lines exist.""" + try: + parsed = parse_graph_citation(citation) + except ContextError: + return False + entry = self._blobs.get(parsed.path) + if entry is None: + return False + if parsed.start_line is None: + return True + claimed = parsed.end_line or parsed.start_line + return claimed <= self._lines(entry) + + def _lines(self, entry: lifecycle.TrackedEntry) -> int: + cached = self._counts.get(entry.blob) + if cached is not None: + return cached + counted = self._count(entry) + self._counts[entry.blob] = counted + return counted + + def _count(self, entry: lifecycle.TrackedEntry) -> int: + """Lines in one blob of the bound commit, read as a stream. + + A blob is read through ``git cat-file``, never off the working tree: + the file at that path today may be a different file, or not exist. An + unreadable blob counts as zero lines, which makes every claim on it + unresolved rather than silently accepted. + """ + if entry.size == 0: + return 0 + try: + process = subprocess.Popen( + ["git", "-C", str(self._repository), "--no-optional-locks", + "cat-file", "blob", entry.blob], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=lifecycle.git_environment(), + ) + except (OSError, ValueError): + return 0 + lines = 0 + trailing = False + try: + assert process.stdout is not None + while True: + chunk = process.stdout.read(_BLOB_CHUNK_BYTES) + if not chunk: + break + lines += chunk.count(b"\n") + trailing = not chunk.endswith(b"\n") + except OSError: + return 0 + finally: + if process.stdout is not None: + try: + process.stdout.close() + except OSError: + pass + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: # pragma: no cover - git does not hang on a closed pipe + process.kill() + process.wait() + if process.returncode != 0: + return 0 + # A file whose last line has no terminator still has that line. + return lines + 1 if trailing else lines + + +def _relation_text(question: str, item: Relation) -> str: + """One sentence of metadata about a relationship. Never indexed content. + + Names, paths and relationship kinds only: everything here is already in the + citations beside it, so the prose adds no claim a recipient cannot check. + + The sentence states exactly the one edge ``via`` carries, between its own + two endpoints. Where the walk reached that edge from is a separate clause, + ``reached from``, so a recipient reads a transitive result as a path and + never as a direct relationship the graph does not assert. + """ + # The provider's own word for the relationship, not this module's grouping + # of it: an ``implements`` edge is filtered as ``inherits`` but must read as + # "implements", and an LLM-extracted relation outside the mapped set must + # read as itself rather than as the ``related`` bucket it was filed under. + verb = item.via.relation + if item.via.source == item.node.id: + subject, object_ = item.node.name, item.origin.name + else: + subject, object_ = item.origin.name, item.node.name + provenance = "" if item.depth <= 1 else f", reached from {item.seed.name}" + return ( + f"{question}: {subject} {verb} {object_} " + f"({item.via.evidence}, hop {item.depth}{provenance}, {item.node.kind} at {item.node.path})" + ) + + +@dataclass(frozen=True) +class PacketDraft: + """A packet and the metadata-only account of what it left out.""" + + packet: dict[str, Any] = field(repr=False) + summary: dict[str, Any] + + +def _documents( + result: QueryResult, validator: CitationValidator, budget: int +) -> tuple[list[dict[str, Any]], list[str]]: + """One document per relationship, with only citations that actually resolve. + + A relationship whose own location cannot be confirmed against the bound + commit is dropped rather than downgraded: the packet's whole claim is that + its citations point at the immutable tree, and evidence that cannot be + pointed at is not weaker evidence, it is none. + + ``budget`` is the selected policy's document allowance. The effective + ceiling is the smaller of it and this adapter's own ``MAX_DOCUMENTS``, so a + policy may only tighten what one packet carries, never lift the adapter + limit. Answering past the policy's budget is not an option: the contract + refuses such a packet at delivery, which turns an honestly truncated answer + into no answer at all. + """ + omissions: list[str] = [] + documents: list[dict[str, Any]] = [] + limit = min(MAX_DOCUMENTS, budget) + dropped = False + unvalidated = False + citations_used = 0 + for item in result.relations: + if len(documents) >= limit: + dropped = True + break + # Both endpoints of the edge the sentence states, never the seed the + # walk started from: a citation is where a recipient goes to check the + # claim, and the claim is about these two nodes. Keyed by citation and + # first-write-wins, so a self-referential relationship cites one + # location once, in a fixed order. + # A sourceless stub contributes no citation at all -- it is an + # unresolved cross-file reference, which is the one endpoint shape the + # pinned extractor emits with no location to point at. It counts as an + # endpoint the bound commit does not confirm, same as a path the census + # does not carry. + endpoints: dict[str, GraphNode] = {} + located = 0 + for endpoint in (item.node, item.origin): + citation = endpoint.citation + if citation is None: + continue + located += 1 + endpoints.setdefault(citation, endpoint) + cited = [(citation, f"{endpoint.kind} {endpoint.name}") + for citation, endpoint in endpoints.items() if validator.validate(citation)] + if len(cited) != len(endpoints) or located < 2: + # The graph claimed a location the bound commit does not carry. + # That is the provider disagreeing with the immutable tree, which + # a recipient must be told about even when the relationship keeps + # a second citation that does check out. + unvalidated = True + # The call site itself, titled apart from either endpoint so a + # recipient can tell where the relationship was written from the two + # places it connects. Missing is not unvalidated -- a call site with no + # location is the provider stating none, exactly like a sourceless + # stub's endpoint citation above -- but a stated one the bound commit + # does not carry is the same disagreement an endpoint's is, and must + # never appear as though it were verified. + call_site = item.via.citation + if call_site is not None: + if validator.validate(call_site): + if call_site not in {source for source, _ in cited}: + cited.append((call_site, f"call site: {item.via.relation}")) + else: + unvalidated = True + cited = cited[:MAX_CITATIONS_PER_DOCUMENT] + if not cited: + dropped = True + continue + if citations_used + len(cited) > MAX_GRAPH_CITATIONS: + dropped = True + break + citations_used += len(cited) + documents.append({ + "text": _relation_text(result.question, item), + "confidence": EVIDENCE_CONFIDENCE[item.via.evidence], + "source_kind": "local_repository_graph", + # Each citation is titled with the node it actually points at, so a + # two-endpoint relationship does not label the endpoint it came + # from with the name of the one it reached. + "citations": [ + {"source": source, "title": title} for source, title in cited + ], + }) + if unvalidated: + omissions.append("provider_warning") + if dropped: + omissions.append("document_limit") + return documents, omissions + + +def build_packet( + result: QueryResult, + *, + validator: CitationValidator, + envelope: Mapping[str, Any], + policy: Mapping[str, Any], + repository: str, + work_item: str, + built_at: str, + completeness: str, + now: datetime | None = None, +) -> PacketDraft: + """Turn one traversal into a revision-bound packet a recipient may read. + + The packet binds the graph's commit as ``source_revision`` and the + generation as its provenance, so a consumer that asked about a different + revision resolves ``stale`` at delivery without decoding the payload. The + binding is the authorization envelope's, unchanged: this module decides + what the evidence is, never who may read it. What it does check is that the + two describe one graph -- an envelope authorizing a generation the + traversal did not read is refused rather than reconciled. + """ + current = now or datetime.now(timezone.utc) + if current.tzinfo is None: + raise ContextError("context packet time must include a timezone") + limits = normalize_policy(policy) + if limits is None: + raise ContextError("context policy is not configured") + connection = validate_connection(envelope, now=current) + if connection["kind"] != "repository": + raise ContextError("local graph evidence requires a repository-kind connection") + # The binding names one generation, and the freshness rules the delivery + # contract enforces all read it: a rebuilt graph is refused because the + # published generation moved. Copying the envelope's word for it would make + # that check vacuous whenever the traversal came from a different + # generation than the one authorized -- the packet would claim provenance + # it does not have, and still pass every later comparison. The envelope is + # not rewritten to match, because it is an authorization this module did + # not mint: the disagreement is the refusal. + if connection["generation"] != result.generation: + raise ContextError("local graph evidence is not from the authorized graph generation") + if completeness not in (lifecycle.COMPLETE, lifecycle.PARTIAL): + raise ContextError("unsupported local graph completeness") + if not result.resolved: + raise ContextError("local graph query resolved no symbol or path to cite") + documents, dropped = _documents(result, validator, limits["max_documents"]) + if not documents: + raise ContextError("local graph query produced no citable evidence") + omissions = list(dict.fromkeys([ + *result.omissions, + *dropped, + # The generation's own admission, carried through rather than + # recomputed: a partial build's answer is partial however complete this + # traversal was. + *(["provider_partial"] if completeness == lifecycle.PARTIAL else []), + ])) + truncated = result.truncated or "document_limit" in omissions + # ``provider_partial`` covers both of its sources -- the generation's own + # partial build, added just above, and a traversal that read a relationship + # whose far end the document never declared. Neither is truncation, and a + # packet carrying either must not call itself complete. + packet_completeness = "partial" if truncated or "provider_partial" in omissions else "complete" + expiry = min( + _timestamp(connection["expires_at"]), + current + timedelta(seconds=limits["max_age_seconds"]), + ) + packet = { + "schema": PACKET_SCHEMA, + "capability_version": CAPABILITY_VERSION, + "provider": connection["provider"], + "kind": "repository", + "retrieved_at": current.isoformat(), + "source_revision": result.commit, + "source_built_at": built_at, + "completeness": packet_completeness, + "truncated": truncated, + "omissions": omissions, + "documents": documents, + "binding": { + **{key: connection[key] for key in ("connection", "generation", "identity", "recipients")}, + "repository": repository, + "work_item": work_item, + "policy_version": limits["policy_version"], + "expires_at": expiry.isoformat(), + }, + } + summary = { + **result.shareable_summary(), + "graph_generation": result.generation, + "documents": len(documents), + "completeness": packet_completeness, + "truncated": truncated, + "omissions": omissions, + "recipients": list(connection["recipients"]), + } + return PacketDraft(packet=packet, summary=summary) + + +#: What a caller does next when the graph cannot answer. ``required_unavailable`` +#: is the blocking outcome and ``optional_unavailable`` the degrading one; the +#: words match ``context_prepare`` so a caller branches on one vocabulary. +AVAILABLE = "available" +REQUIRED_UNAVAILABLE = "required_unavailable" +OPTIONAL_UNAVAILABLE = "optional_unavailable" + + +@dataclass(frozen=True) +class GraphContext: + """The answer, or the reason there is none and what that means for the work.""" + + status: str + summary: dict[str, Any] + packet: dict[str, Any] | None = field(default=None, repr=False) + + @property + def dependent_work(self) -> str: + return "paused" if self.status == REQUIRED_UNAVAILABLE else "usable" + + @property + def exit_code(self) -> int: + return 1 if self.status == REQUIRED_UNAVAILABLE else 0 + + +def _unavailable(required: bool, reason: str, detail: Mapping[str, Any] | None = None) -> GraphContext: + return GraphContext( + status=REQUIRED_UNAVAILABLE if required else OPTIONAL_UNAVAILABLE, + summary={ + "schema": QUERY_SCHEMA, + "status": REQUIRED_UNAVAILABLE if required else OPTIONAL_UNAVAILABLE, + "reason": reason, + "dependent_work": "paused" if required else "usable", + "next_action": ( + "Refresh or build the local graph for this revision, then retry; dependent work is blocked." + if required else + "Continue with ordinary repository tools; optional graph context was not used." + ), + **(dict(detail) if detail else {}), + }, + ) + + +def graph_context( + repository: Path, + *, + question: str, + target: str, + envelope: Mapping[str, Any], + policy: Mapping[str, Any], + context_repository: str, + work_item: str, + root: Path | None = None, + revision: str = "HEAD", + depth: int | None = None, + node_budget: int = DEFAULT_NODE_BUDGET, + now: datetime | None = None, +) -> GraphContext: + """Answer one question from the published generation, or say why not. + + This is the whole decision in one call, and the order matters. Usability is + settled first, against ``graph_status``, which is what binds the revision + and verifies the artifact digest: a graph that is absent, stale, partial, + corrupt or oversized never reaches a traversal. Only then is the graph + read, queried, and turned into evidence. + + Required context that is unavailable blocks the dependent work. Optional + context that is unavailable returns ``optional_unavailable`` with + ``dependent_work`` usable, which is the contract's way of saying: carry on + with ordinary repository tools. Neither outcome raises, because "no graph" + is a normal state of an opt-in feature, not a failure of the caller. + """ + limits = normalize_policy(policy) + if limits is None: + raise ContextError("context policy is not configured") + required = bool(limits["required"]) + if question not in QUESTIONS: + raise ContextError("unsupported local graph question") + state = lifecycle.GraphStateRoot(repository, root=root) + status = lifecycle.graph_status(repository, root=root, revision=revision) + if not status.usable: + return _unavailable(required, status.state, {"detail": status.detail}) + manifest = status.manifest + assert manifest is not None # a usable status always carries one + try: + graph = read_graph(state, status) + census = lifecycle.read_tracked_census(state.repository, manifest.commit) + except ContextError as error: + return _unavailable(required, "unreadable", {"detail": str(error)}) + result = run_query(graph, question=question, target=target, depth=depth, node_budget=node_budget) + if not result.resolved: + return _unavailable(required, "unresolved", {"detail": "the graph carries no such symbol or path"}) + try: + draft = build_packet( + result, + validator=CitationValidator(state.repository, census), + envelope=envelope, + policy=policy, + repository=context_repository, + work_item=work_item, + built_at=manifest.built_at, + completeness=manifest.completeness, + now=now, + ) + except ContextError as error: + return _unavailable(required, "uncitable", {"detail": str(error)}) + return GraphContext( + status=AVAILABLE, + summary={"schema": QUERY_SCHEMA, "status": AVAILABLE, "dependent_work": "usable", **draft.summary}, + packet=draft.packet, + ) + + +__all__: Sequence[str] = ( + "AVAILABLE", + "CitationValidator", + "CodeGraph", + "DEFAULT_NODE_BUDGET", + "EVIDENCE_CONFIDENCE", + "GRAPH_CONFIDENCES", + "GRAPH_FILE_TYPES", + "GRAPH_MEMBER", + "GRAPH_RELATIONS", + "GraphContext", + "GraphEdge", + "GraphNode", + "MAX_NODE_BUDGET", + "OPTIONAL_UNAVAILABLE", + "PacketDraft", + "QUESTIONS", + "QUERY_SCHEMA", + "QueryResult", + "REQUIRED_UNAVAILABLE", + "Relation", + "build_packet", + "graph_context", + "load_graph", + "read_graph", + "run_query", +) diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index aee2504b..b31bddc1 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -3,10 +3,12 @@ from __future__ import annotations import hashlib +import re import uuid from pathlib import Path from typing import Any, Mapping +from . import context_graph_connection as graph_connection from . import context_review, context_session from .claude_audit_pr import _decision_authorities_for_repo from .context_contract import ContextError, ContextRequest @@ -21,7 +23,7 @@ reserve_attachment, retire_attachment, ) -from .context_packets import load_authorized +from .context_packets import consuming_revision, load_authorized from .context_store import ContextStore from .participants import PARTICIPANTS, participant_id from .provider_runners import fetch_issue_comments, fetch_pull_request, post_pr_comment @@ -30,6 +32,7 @@ ATTACH_SCHEMA = "code_mower.contextSessionAttach.v1" +_HEAD_SHA = re.compile(r"[a-f0-9]{40}\Z") def _workflow_key(record: Mapping[str, Any]) -> str: @@ -63,6 +66,12 @@ def _remote_input( head = pull["head"]["sha"] except (KeyError, TypeError): raise ContextError("GitHub did not return a valid pull request head") from None + # Malformed remote data (missing, null, wrong-type, shortened, or + # non-hex) is not authoritative head movement: refuse before any + # comparison or cleanup ever sees it, rather than let it masquerade as a + # moved or matching head. + if not isinstance(head, str) or not _HEAD_SHA.fullmatch(head): + raise ContextError("GitHub did not return a valid pull request head") current = context_review.latest_input( fetch_issue_comments(repository, pr, token=token), authorities=authorities, ) @@ -111,6 +120,7 @@ def _reserve_for_record( packet_store: ContextStore, record: Mapping[str, Any], *, + repo_path: Path, backend: Any, ) -> dict[str, Any]: try: @@ -125,6 +135,10 @@ def _reserve_for_record( pr=record["pr"], head=record["head"], revision=record["revision"], + # This checkout's actual consuming revision, not the remote head + # alone; a repository connection fails closed if the checkout has + # moved, both for a fresh reservation and a pending/uncertain retry. + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: @@ -166,7 +180,13 @@ def _discard_stale_attachment( packet_store: ContextStore, record: Mapping[str, Any], ) -> dict[str, Any]: - """Clear one binding only after the live PR head has moved beyond it.""" + """Clear one binding only after the live PR head has moved beyond it. + + Retiring the packet-store binding and updating the association are two + separate writes; either alone is retry-safe (a missing binding is a + no-op for ``retire_attachment``), so an interruption between them is + resolved by simply calling this again with the still-saved record. + """ retire_attachment( packet_store, record["connection"], record["packet"], record["revision"], ) @@ -179,6 +199,54 @@ def _discard_stale_attachment( "attachment_state": "none", }, ) + + +def _abandon_and_clear( + association_store: ContextStore, + packet_store: ContextStore, + record: Mapping[str, Any], +) -> None: + """Best-effort return of a not-yet-published intent to prepared/none. + + Used only when ``record["revision"]`` is known to have not reached + ``_finish_publication``'s remote publication check, so any stored + delivery binding for it is never published and safe to abandon. Never + used to blanket-clear a pre-existing pending/uncertain retry -- those + have already durably transitioned past ``reserving`` and may be the + GitHub-accepted state a lost response only looked like it missed. Both + the same-call rollback after a failed fresh reservation and a later + attach's recognition of a still-``reserving`` record call this; in + either case the identity being cleared is proven pre-publication by the + caller, never inferred here from a missing binding alone. + ``record_failure`` may already have advanced the association generation, + so the session is re-read here rather than trusting a stale one; if a + concurrent operation has already moved the saved revision on, cleanup is + skipped rather than overwriting that operation's result. A + ``ContextError`` or storage exception from either step does not prove no + local write occurred, so on failure the saved intent is left in place + instead of being claimed cleared -- the caller's own error still reports + the original failure. + """ + try: + current = context_session.read(association_store, record["session_id"]) + if current is None or current["revision"] != record["revision"]: + return + abandon_attachment( + packet_store, current["connection"], current["packet"], current["revision"], + ) + context_session.update( + association_store, + current["session_id"], + expected_generation=current["generation"], + changes={ + "stage": "prepared", "pr": None, "head": None, "revision": None, + "attachment_state": "none", + }, + ) + except (ContextError, OSError): + pass + + def attach_session( association_store: ContextStore, packet_store: ContextStore, @@ -210,11 +278,34 @@ def attach_session( record = current_record head, current = _remote_input(record["repo"], pr, token=token, authorities=authorities) + if record["attachment_state"] == "reserving": + # A ``reserving`` identity has, by construction, never reached + # ``_finish_publication``: it is written before ``reserve_attachment`` + # is even called, and only ever advances to ``pending`` -- still + # strictly before any GitHub write -- after that reservation + # durably succeeds. Finding it still ``reserving`` on a later + # attach therefore always means the same call that minted it + # stopped somewhere before, during, or just after reservation, + # with no public write possible, so it is always safe to abandon + # here regardless of the requested pull request. This never + # completes a fresh reattachment inline -- that would risk a + # public write racing this recovery -- so the caller reruns + # attach once cleanup below is done. + _abandon_and_clear(association_store, packet_store, record) + raise ContextError( + "a saved publication attempt did not complete; rerun attach to finish reconciliation" + ) + if record["attachment_state"] == "published": if record["pr"] != pr: raise ContextError("this session is already attached to a different pull request") - binding = read_binding(packet_store, record["revision"]) if record["head"] == head: + # Only the same-head path needs the existing binding read. + # A moved head retires by identity below without ever + # reading it, so an interrupted prior retirement's missing + # binding is not read here as current evidence; a same-head + # missing binding still fails closed. + binding = read_binding(packet_store, record["revision"]) if current != binding["metadata"]: raise ContextError( "the trusted current input changed; inspect the pull request before replacing it" @@ -228,6 +319,10 @@ def attach_session( head=head, recipient=record["host"] + ":orchestrator", current=current, + # This checkout's actual consuming revision, not the + # remote head alone; a repository connection fails + # closed if the checkout has moved. + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: @@ -239,11 +334,20 @@ def attach_session( if record["attachment_state"] in {"pending", "uncertain"}: if record["pr"] != pr: raise ContextError("a saved attachment intent targets a different pull request") - metadata = _reserve_for_record( - association_store, packet_store, record, backend=backend, - ) - if current == metadata: - if record["head"] == head: + if record["head"] != head: + # Retire the old identity before ever reauthorizing its + # evidence. Reserving first, as below, reauthorizes the + # stale saved head's evidence; a checkout that has since + # moved on could fail that reauthorization before this + # already-safe retirement path ever ran. + record = _discard_stale_attachment( + association_store, packet_store, record, + ) + else: + metadata = _reserve_for_record( + association_store, packet_store, record, repo_path=repo_path, backend=backend, + ) + if current == metadata: mark_published(packet_store, record["connection"], record["revision"]) record = context_session.update( association_store, @@ -255,16 +359,8 @@ def attach_session( }, ) return _report("attached", reused=True, reconciled=True), 0 - record = _discard_stale_attachment( - association_store, packet_store, record, - ) - elif record["attachment_state"] == "uncertain" and not retry_uncertain: - return _report("attachment_uncertain", reused=True), 1 - elif record["head"] != head: - record = _discard_stale_attachment( - association_store, packet_store, record, - ) - elif record["attachment_state"] in {"pending", "uncertain"}: + if record["attachment_state"] == "uncertain" and not retry_uncertain: + return _report("attachment_uncertain", reused=True), 1 return _finish_publication( association_store, packet_store, @@ -281,11 +377,34 @@ def attach_session( expected_generation=record["generation"], changes={ "stage": "prepared", "pr": pr, "head": head, "revision": revision, - "attachment_state": "pending", + "attachment_state": "reserving", }, ) - metadata = _reserve_for_record( - association_store, packet_store, record, backend=backend, + try: + metadata = _reserve_for_record( + association_store, packet_store, record, repo_path=repo_path, backend=backend, + ) + except ContextError: + # This exact intent was just minted in this same call and never + # reached publication, so it is safe to abandon; a pre-existing + # pending/uncertain intent retried above is never cleared this + # way. + _abandon_and_clear(association_store, packet_store, record) + raise + # The reservation itself is now durable and unpublished. Recording + # that as ordinary ``pending`` -- still strictly before + # ``_finish_publication`` ever touches GitHub -- is what lets a later + # attach trust that ``pending`` always names a real, resumable + # reservation, never the ghost of one that was silently abandoned + # underneath it. If this transition itself does not complete (a + # crash, or a storage/generation failure), the record stays + # ``reserving`` and the next attach recognizes and clears it instead + # of replaying a reservation that may already be gone. + record = context_session.update( + association_store, + record["session_id"], + expected_generation=record["generation"], + changes={"attachment_state": "pending"}, ) return _finish_publication( association_store, @@ -308,17 +427,7 @@ def _finish_publication( ) -> tuple[dict[str, Any], int]: head, current = _remote_input(record["repo"], record["pr"], token=token, authorities=authorities) if head != record["head"]: - abandon_attachment( - packet_store, record["connection"], record["packet"], record["revision"], - ) - context_session.update( - association_store, - record["session_id"], - expected_generation=record["generation"], - changes={ - "pr": None, "head": None, "revision": None, "attachment_state": "none", - }, - ) + _abandon_and_clear(association_store, packet_store, record) raise ContextError("pull request head changed before publication; rerun attach") if current == metadata: mark_published(packet_store, record["connection"], record["revision"]) @@ -350,6 +459,12 @@ def _builder_recipient(record: Mapping[str, Any]) -> str: return recipient +def _is_repository_connection(packet_store: ContextStore, name: str) -> bool: + """Whether ``name`` is a local repository graph rather than an organization connection.""" + with packet_store.locked(name) as locked: + return graph_connection.is_graph(locked.read()) + + def deliver_session( association_store: ContextStore, packet_store: ContextStore, @@ -371,7 +486,12 @@ def deliver_session( record["connection"], record["packet"], record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest( + record["repo"], record["work_item"], recipient, + # Before attachment the consuming revision is this + # checkout's, which is what the builder is about to work on. + consuming_revision(repo_path), + ), backend=backend, ) text = render_evidence(packet, record["packet"]) @@ -380,6 +500,15 @@ def deliver_session( head, current = _remote_input( record["repo"], record["pr"], token=token, authorities=authorities, ) + if _is_repository_connection(packet_store, record["connection"]): + # This checkout's own consuming revision, not just the bound + # packet or PR head that ``deliver`` checks below; an + # unresolvable revision (e.g. not a Git checkout) fails closed. + if consuming_revision(repo_path) != head: + raise ContextError( + "repository context evidence is not bound to this " + "checkout's consuming revision" + ) text = deliver( packet_store, record["revision"], @@ -388,6 +517,7 @@ def deliver_session( head=head, recipient=recipient, current=current, + consuming_revision=consuming_revision(repo_path), backend=backend, ).text except ContextError as exc: @@ -437,6 +567,7 @@ def feedback_session( head=head, recipient=recipient, current=current, + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: diff --git a/src/code_mower/context_packets.py b/src/code_mower/context_packets.py index e5050dc9..0988c789 100644 --- a/src/code_mower/context_packets.py +++ b/src/code_mower/context_packets.py @@ -11,6 +11,8 @@ from datetime import datetime, timedelta, timezone from pathlib import Path +from . import context_graph_connection as graph_connection +from . import context_graph_lifecycle as lifecycle from .context_connections import _backend, _state, authorize_locked from .context_contract import ( CAPABILITY_VERSION, PACKET_SCHEMA, ContextError, ContextRequest, ContextRetrievalError, _object, @@ -41,6 +43,26 @@ def request_spec(value, name): "policy": policy} +def consuming_revision(repo_root) -> str | None: + """The commit the *consuming* checkout is at, or ``None`` if it has none. + + This is the revision prepared evidence is for, and it is read from the + checkout doing the work rather than from whichever checkout a connection was + registered against: the two are routinely different commits, and a local + repository graph describing the other one does not describe this work. + + ``None`` rather than a raise, so a connection that has no use for a code + revision -- an organization search, whose sources are documents with their + own versions -- still prepares from a directory that is not a Git checkout. + A repository-kind connection refuses instead of falling back. + """ + try: + commit, _tree = lifecycle.resolve_revision(Path(repo_root)) + except (ContextError, OSError, ValueError): + return None + return commit + + def _handle(value): try: if uuid.UUID(hex=value).hex != value: @@ -107,37 +129,84 @@ def _delete_entry(locked, entry): locked.artifact("p-" + entry["handle"]).delete() -def _request(spec, recipient=None): - return ContextRequest(spec["repository"], spec["work_item"], recipient or spec["recipient"]) +def _request(spec, recipient=None, revision=None): + return ContextRequest(spec["repository"], spec["work_item"], recipient or spec["recipient"], + revision) + + +def _load(store, entry, policy, request, envelope, *, bound_revision=None): + """Load one saved packet, and for a local graph require the consuming revision. + ``bound_revision`` is the commit the *consuming* work is at, resolved by the + same authorization that produced ``envelope``. The shared contract already + computes ``revision_state`` from it; what is decided here is what a + mismatch means. For a local repository graph it is a refusal: the evidence + describes one commit's code, so evidence for commit A handed to work on + commit B is wrong rather than merely old, and the caller's own required or + optional policy then decides whether that pauses or degrades the work. -def _load(store, entry, policy, request, envelope): + An organization connection passes ``None`` and is unaffected. Its source + revision is an external document version that has no reason to equal a code + commit, and requiring one would refuse every organization packet. + """ if entry["reference"] is None: if "failure_reason" in entry: raise ContextRetrievalError(entry["failure_reason"]) raise ContextError("context retrieval did not complete; use an explicit refresh to try again") - return load_packet(private_root=store.root, reference=entry["reference"], policy=policy, - request=request, authorize=lambda: envelope) + packet = load_packet(private_root=store.root, reference=entry["reference"], policy=policy, + request=request, authorize=lambda: envelope) + if bound_revision is not None and packet.revision_state != "matching": + raise ContextError("local graph evidence is not bound to the consuming revision") + return packet -def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): - """Retrieve once, or reauthorize and reuse; never redispatch automatically.""" +def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revision=None): + """Retrieve once, or reauthorize and reuse; never redispatch automatically. + + Which provider answers is the connection's own saved state, read here under + the same lock that guards the retrieval. A local repository graph reaches + the same index, the same packet files, and the same delivery contract as an + organization connection; what differs is only where authorization and + evidence come from, and neither kind can be mistaken for the other because + the saved schema is checked before either path is taken. + + ``revision`` is the commit the *consuming* work is at, and it has no + default: a caller that cannot name it gets a refusal from a repository + graph rather than the registered checkout's ``HEAD``, which is a different + checkout that moves independently. An organization connection never reads + it, so the same default costs it nothing. + """ spec = request_spec(spec, name) policy = spec["policy"] started = time.monotonic() - backend = backend or _backend() with store.locked(name, timeout_seconds=policy["timeout_seconds"]) as locked: + local = graph_connection.is_graph(locked.read()) left = policy["timeout_seconds"] - (time.monotonic() - started) if left <= 0: raise ContextError("context retrieval deadline exceeded before authorization") - envelope = authorize_locked(locked, name, backend, timeout_seconds=min(left, 30)) + bound = None + if local: + if revision is None: + # Default-deny rather than fall back to the registered + # checkout's ``HEAD``: that fallback is exactly how evidence for + # one commit reaches work on another. + raise ContextError("local graph context requires the consuming checkout revision") + envelope, bound = graph_connection.authorized_revision( + locked, name, root=store.root, revision=revision, + ) + else: + # Deferred until the connection kind is known: a local graph must + # not require the optional provider SDK to be installed at all. + backend = backend or _backend() + envelope = authorize_locked(locked, name, backend, timeout_seconds=min(left, 30)) if spec["repository"] not in envelope["repositories"] or spec["recipient"] not in envelope["recipients"]: raise ContextError("context connection does not authorize this repository or recipient") fingerprint = _key({k: v for k, v in spec.items() if k != "recipient"}) index_file, index = _index(locked) old = next((entry for entry in index["entries"] if entry["key"] == fingerprint), None) if old is not None and not refresh: - packet = _load(store, old, policy, _request(spec), envelope) + packet = _load(store, old, policy, _request(spec, revision=bound), envelope, + bound_revision=bound) return {**packet.shareable_summary(), "status": "available", "packet_handle": old["handle"], "reused": True, "usage": old["usage"]} # Reserve before any paid/read tool call. Restarting a failed attempt @@ -155,31 +224,45 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): left = policy["timeout_seconds"] - (time.monotonic() - started) if left <= 0: raise ContextError("context retrieval deadline exceeded; no search was sent") - state = _state(locked.read(), name) - credentials = locked.vault.get(state["credential_id"]) + state = graph_connection.saved_state(locked.read(), name) if local else _state(locked.read(), name) + credentials = None if local else locked.vault.get(state["credential_id"]) try: try: - result = backend.retrieve(credentials, spec["query"], spec["source"], policy, timeout_seconds=left) + if local: + # The local graph mints its own packet: it decides what the + # evidence is, and the envelope above decides who may read it. + packet_data = graph_connection.retrieve( + state, spec, envelope=envelope, root=store.root, revision=revision, + ) + else: + result = backend.retrieve(credentials, spec["query"], spec["source"], policy, timeout_seconds=left) except Exception as exc: + # Either provider failing to produce evidence is a retrieval + # failure, told apart below from storage and packet validation. raise ContextRetrievalError( exc.reason if isinstance(exc, ContextRetrievalError) else "retrieval_failed" ) from None - now = datetime.now(timezone.utc) - expiry = min(_timestamp(envelope["expires_at"]), now + timedelta(seconds=policy["max_age_seconds"])) - packet_data = {"schema": PACKET_SCHEMA, "capability_version": CAPABILITY_VERSION, - "provider": "coworker", "kind": "organization", "retrieved_at": now.isoformat(), - **{key: result[key] for key in ("documents", "completeness", "truncated", "source_revision", "source_built_at", "omissions")}, - "binding": {**{key: envelope[key] for key in ("connection", "generation", "identity", "recipients")}, - "repository": spec["repository"], "work_item": spec["work_item"], - "policy_version": policy["policy_version"], "expires_at": expiry.isoformat()}} + usage = None + if not local: + now = datetime.now(timezone.utc) + expiry = min(_timestamp(envelope["expires_at"]), now + timedelta(seconds=policy["max_age_seconds"])) + packet_data = {"schema": PACKET_SCHEMA, "capability_version": CAPABILITY_VERSION, + "provider": "coworker", "kind": "organization", "retrieved_at": now.isoformat(), + **{key: result[key] for key in ("documents", "completeness", "truncated", "source_revision", "source_built_at", "omissions")}, + "binding": {**{key: envelope[key] for key in ("connection", "generation", "identity", "recipients")}, + "repository": spec["repository"], "work_item": spec["work_item"], + "policy_version": policy["policy_version"], "expires_at": expiry.isoformat()}} + usage = result["usage"] locked.artifact("p-" + entry["handle"]).write(packet_data) raw = json.dumps(packet_data, allow_nan=False, separators=(",", ":")).encode() entry["reference"] = {"path": ".p-" + entry["handle"] + ".json", "sha256": hashlib.sha256(raw).hexdigest()} - packet = _load(store, entry, policy, _request(spec), envelope) - entry["usage"] = result["usage"] + packet = _load(store, entry, policy, _request(spec, revision=bound), envelope, + bound_revision=bound) + entry["usage"] = usage index_file.write(index) _index(locked) - locked.write({**state, "capability_status": {"search": "available", "memory": "available"}}) + if not local: + locked.write({**state, "capability_status": {"search": "available", "memory": "available"}}) except Exception as exc: # Storage and local packet validation are not provider failures. failure = ContextRetrievalError( @@ -192,7 +275,9 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): try: index_file.write(index) locked.artifact("p-" + entry["handle"]).delete() - locked.write({**state, "capability_status": {"search": "unavailable", "memory": "unavailable"}}) + if not local: + # A local graph connection keeps no provider capability state. + locked.write({**state, "capability_status": {"search": "unavailable", "memory": "unavailable"}}) except (OSError, ContextError): entry["failure_reason"] = "storage_unavailable" try: @@ -205,16 +290,44 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): "reused": False, "usage": entry["usage"]} -def load_authorized(store, name, handle, policy, request: ContextRequest, *, backend=None): - """Every participant replay obtains a new online authorization under lock.""" +def load_authorized(store, name, handle, policy, request: ContextRequest, *, backend=None, + revision=None): + """Every participant replay obtains a new authorization under lock. + + For an organization connection that is a fresh online check. For a local + repository graph it is a fresh read of current local state: the published + generation for the *consuming* revision. Either way the envelope is minted + here and now, so a packet whose graph was rebuilt or whose revision has + moved on is refused by the shared contract rather than replayed. + + The consuming revision travels on the request the caller already builds -- + ``ContextRequest.revision`` -- and ``revision`` is the same value for the + callers that hold it without holding a request, such as an attachment that + knows only the trusted current PR head. Neither is defaulted to ``HEAD`` + for a graph: a replay that cannot name the revision it is for is refused, + because the checkout the graph was registered from moves independently of + the work consuming the evidence. + """ _handle(handle) with store.locked(name) as locked: - envelope = authorize_locked(locked, name, backend or _backend()) + bound = None + if graph_connection.is_graph(locked.read()): + consuming = request.revision or revision + if consuming is None: + raise ContextError("local graph context requires the consuming checkout revision") + envelope, bound = graph_connection.authorized_revision( + locked, name, root=store.root, revision=consuming, + ) + # Resolved, so a symbolic consuming revision is compared as the + # commit it names rather than as the word the caller typed. + request = ContextRequest(request.repository, request.work_item, request.recipient, bound) + else: + envelope = authorize_locked(locked, name, backend or _backend()) _file, index = _index(locked) entry = next((entry for entry in index["entries"] if entry["handle"] == handle), None) if entry is None: raise ContextError("context packet is missing or was invalidated") - return _load(store, entry, policy, request, envelope) + return _load(store, entry, policy, request, envelope, bound_revision=bound) def main(argv=None): @@ -230,7 +343,16 @@ def main(argv=None): try: spec = request_spec(strict_json(sys.stdin.buffer.read(262_145)), args.connection) required = spec["policy"]["required"] - result = fetch(ContextStore(args.state_dir), args.connection, spec, refresh=args.refresh) + # The consuming checkout is the one this command was run from, not the + # one a connection was registered against. Leaving ``fetch`` to default + # to ``HEAD`` resolves that word in the *registered* graph checkout, so a + # graph for commit A could answer work at commit B -- the exact fallback + # the guided route already refuses. ``None`` when the caller is not a Git + # checkout at all: that refuses a repository graph here (it cannot name + # the revision its evidence would be for) and is ignored by an + # organization connection, whose sources version independently of code. + result = fetch(ContextStore(args.state_dir), args.connection, spec, refresh=args.refresh, + revision=consuming_revision(Path.cwd())) code = 0 except (ContextError, OSError, ValueError) as exc: result = {"status": "required_unavailable" if required else "optional_unavailable", diff --git a/src/code_mower/context_prepare.py b/src/code_mower/context_prepare.py index 4d35a56b..a15b3464 100644 --- a/src/code_mower/context_prepare.py +++ b/src/code_mower/context_prepare.py @@ -9,6 +9,7 @@ from . import context_packets, context_session, work_orders from .context_contract import ContextError, ContextRequest, _text +from .context_packets import consuming_revision from .context_delivery import SUPPORTED_HOSTS from .context_store import ContextStore from .participants import PARTICIPANTS, participant_id @@ -224,7 +225,7 @@ def prepare( work_order=record["work_order"], reused=True, ), 0 - if record["attachment_state"] in {"pending", "uncertain"}: + if record["attachment_state"] in {"reserving", "pending", "uncertain"}: if refresh: raise ContextError( "reconcile the saved attachment before refreshing or start a new session" @@ -242,6 +243,10 @@ def prepare( reused=True, ), 0 + # Read once, before any retrieval or replay: every load below is for the + # work this checkout is at, and a checkout that moves mid-preparation must + # not have one packet authorized against two commits. + revision = consuming_revision(repo_root) explicit_query = query is not None effective_query = _text( query if explicit_query else DEFAULT_QUERY_PREFIX + record["work_item"], @@ -289,7 +294,7 @@ def prepare( record["connection"], record["packet"], record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest(record["repo"], record["work_item"], recipient, revision), backend=backend, ) except ContextError as exc: @@ -381,7 +386,7 @@ def prepare( record["connection"], packet_handle, record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest(record["repo"], record["work_item"], recipient, revision), backend=backend, ) except ContextError as exc: @@ -409,6 +414,7 @@ def prepare( }, backend=backend, refresh=refresh, + revision=revision, ) except ContextError as exc: context_session.record_failure(association_store, record, exc) diff --git a/src/code_mower/context_session.py b/src/code_mower/context_session.py index 54c097ed..4a4599ad 100644 --- a/src/code_mower/context_session.py +++ b/src/code_mower/context_session.py @@ -19,7 +19,14 @@ ASSOCIATION_SCHEMA = "code_mower.contextSession.v1" STATUS_SCHEMA = "code_mower.contextSessionStatus.v1" STAGES = frozenset(("selected", "preparing", "prepared", "attached", "reviewed")) -ATTACHMENT_STATES = frozenset(("none", "pending", "published", "uncertain")) +ATTACHMENT_STATES = frozenset( + ("none", "reserving", "pending", "published", "uncertain") +) +#: ``reserving`` is the durable, pre-publication marker a freshly minted +#: attachment identity holds before ``reserve_attachment`` is even called. +#: It proves no GitHub write has begun, so it is always safe to abandon; +#: only a successful transition to ``pending`` (still before any GitHub +#: write) records that the reservation itself durably exists. CONTEXT_STATES = frozenset( ("unchecked", "ready", "expired", "authorization_failed", "unavailable") ) @@ -232,7 +239,7 @@ def validate(value: Mapping[str, Any]) -> dict[str, Any]: attachment_fields = (pr, head, revision) if attachment == "none" and any(item is not None for item in attachment_fields): raise ContextError("private session context revision has no attachment") - if attachment in {"pending", "uncertain"} and ( + if attachment in {"reserving", "pending", "uncertain"} and ( value["stage"] != "prepared" or any(item is None for item in attachment_fields) ): raise ContextError("private session context attachment intent is incomplete") @@ -435,6 +442,18 @@ def status(record: Mapping[str, Any] | None, *, lease_live: bool) -> dict[str, A "stage": "not_configured", "dependent_work": "usable", "owner_action": False, "next_action": "Continue the ordinary workflow or configure an optional context connection.", } + if record["attachment_state"] == "reserving": + # A ``reserving`` intent has never reached a GitHub write, so attach + # can always safely reconcile it without reauthorization; that takes + # precedence over any ``context_state`` failure recorded against a + # prior generation of this same record, unlike ``pending``/ + # ``uncertain`` below, whose failures may still need owner action. + return { + "schema": STATUS_SCHEMA, "selected": True, "configured": True, + "stage": "attachment_pending", "dependent_work": "paused", + "owner_action": False, + "next_action": "Rerun attach to reconcile or finish the saved publication intent.", + } if record["context_state"] in {"expired", "authorization_failed", "unavailable"}: expired = record["context_state"] == "expired" authorization = record["context_state"] == "authorization_failed" diff --git a/src/code_mower/devin_work_orders.py b/src/code_mower/devin_work_orders.py index 8bc5eb7a..8601ab01 100644 --- a/src/code_mower/devin_work_orders.py +++ b/src/code_mower/devin_work_orders.py @@ -253,10 +253,14 @@ class PacketContext: handle: str = field(repr=False) policy: dict = field(repr=False) backend: object = field(default=None, repr=False) + #: The commit this order's work consumes. Repository-kind evidence is + #: authorized against it and refuses when it is absent, because a graph + #: describing another commit does not describe this order. + revision: str | None = field(default=None, repr=False) def packet_context(store: ContextStore, name: str, handle: str, policy, *, order: WorkOrder, - backend=None) -> PacketContext: + backend=None, revision: str | None = None) -> PacketContext: """Bind one authorized packet to the hosted builder before its PR exists. The packet request is derived from the work order (repository and its @@ -264,17 +268,27 @@ def packet_context(store: ContextStore, name: str, handle: str, policy, *, order trusted policy's ``required`` flag must agree with the order's declared context policy. Each render performs a new online authorization for ``devin:builder``; nothing is cached or written. + + ``revision`` is the commit this order's work consumes. It is the one input + here a caller does supply, because there is nothing else to derive it from: + a hosted order has no PR head yet and this process is not the checkout doing + the work. Repository-kind evidence -- the local graph route among it -- is + authorized against that commit on every render, so without it such evidence + refuses rather than answering for whichever commit the connection happens to + be registered at. Organization-kind evidence, whose sources version + themselves, is unaffected and still prepares with ``None``. """ try: normalized = normalize_policy(policy) _handle(handle) + bound = _text(revision, maximum=200) if revision is not None else None except ContextError: raise RemoteError("invalid_request") from None if (normalized is None or order.context_policy == "none" or normalized["required"] != (order.context_policy == "required") or type(store) is not ContextStore or not isinstance(name, str)): raise RemoteError("invalid_request") - return PacketContext(store, name, handle, normalized, backend) + return PacketContext(store, name, handle, normalized, backend, bound) def _github_call(method, *args, **kwargs): @@ -371,7 +385,8 @@ def _evidence(order, context): if (order.context_policy == "none" or type(context) is not PacketContext or type(context.store) is not ContextStore): raise RemoteError("context_binding_mismatch") - request = ContextRequest(order.repository, order.work_item, CONTEXT_RECIPIENT) + request = ContextRequest(order.repository, order.work_item, CONTEXT_RECIPIENT, + context.revision) try: packet = load_authorized(context.store, context.name, context.handle, context.policy, request, backend=context.backend) diff --git a/src/code_mower/package_manifest.py b/src/code_mower/package_manifest.py index ba9dcb90..b085e366 100644 --- a/src/code_mower/package_manifest.py +++ b/src/code_mower/package_manifest.py @@ -50,11 +50,21 @@ "src/code_mower/context_graph_lifecycle.py", "core", ), + ( + "src/code_mower/context_graph_query.py", + "src/code_mower/context_graph_query.py", + "core", + ), ( "src/code_mower/context_graph_command.py", "src/code_mower/context_graph_command.py", "core", ), + ( + "src/code_mower/context_graph_connection.py", + "src/code_mower/context_graph_connection.py", + "core", + ), ("src/code_mower/productivity_report.py", "src/code_mower/productivity_report.py", "core"), ("tools/code_mower_requirements.txt", "requirements/requirements.txt", "tooling"), ("tools/code_mower_calibration.py", "src/code_mower/code_mower_calibration.py", "core"), @@ -598,6 +608,7 @@ ("docs/slack-contract.md", "docs/slack-contract.md", "doc"), ("docs/graphify-evaluation.md", "docs/graphify-evaluation.md", "doc"), ("docs/context-graph-lifecycle.md", "docs/context-graph-lifecycle.md", "doc"), + ("docs/context-graph-queries.md", "docs/context-graph-queries.md", "doc"), ("docs/context-connections.md", "docs/context-connections.md", "doc"), ("docs/context-setup.md", "docs/context-setup.md", "doc"), ("docs/v130-release-notes.md", "docs/v130-release-notes.md", "doc"), diff --git a/src/code_mower/session.py b/src/code_mower/session.py index b24ce9b8..c80ec3c1 100644 --- a/src/code_mower/session.py +++ b/src/code_mower/session.py @@ -13,7 +13,10 @@ from pathlib import Path from typing import Any, Mapping -from . import context_guided, context_prepare, context_session, remote_session_cli, session_current, session_lease +from . import ( + context_graph_query, context_guided, context_prepare, context_session, + remote_session_cli, session_current, session_lease, +) from .config import ConfigError, _format_issues, load_config, validate_config from .context_contract import ContextError, normalize_policy from .context_store import ContextStore @@ -410,7 +413,12 @@ def _run_context_command(args: argparse.Namespace) -> tuple[dict[str, Any], int] ) return {"private_text": text}, 0 tracker = source.get("tracker") - retrieval_source = ( + # ``--question`` is the local repository graph's retrieval source: that + # connection answers one bounded question about a named symbol or path, and + # the question is the operator's to choose. It takes precedence over the + # tracker-derived source, which only means anything to an organization + # search; a checkout cannot be connected to both at one connection name. + retrieval_source = getattr(args, "question", None) or ( "jira" if isinstance(tracker, Mapping) and tracker.get("kind") == "jira_cloud" else None ) return context_prepare.prepare( @@ -512,6 +520,10 @@ def main(argv: list[str] | None = None) -> int: "--query-stdin", action="store_true", help="read a private query override from stdin; the selected work item is the default", ) + context_prepare_parser.add_argument( + "--question", choices=context_graph_query.QUESTIONS, + help="bounded question for a local repository graph connection; the query names the symbol or path", + ) context_prepare_parser.add_argument("--work-order-body-file", type=Path) context_prepare_parser.add_argument("--title") context_prepare_parser.add_argument( diff --git a/tests/test_context_audit.py b/tests/test_context_audit.py index e296aace..57814ff7 100644 --- a/tests/test_context_audit.py +++ b/tests/test_context_audit.py @@ -5,17 +5,25 @@ import json import os import subprocess +import tempfile import unittest from contextlib import ExitStack, redirect_stderr +from pathlib import Path from unittest import mock from code_mower import claude_audit_pr as claude from code_mower import codex_audit_pr as codex -from code_mower import context_audit, context_packets +from code_mower import context_audit, context_delivery, context_packets +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle +from code_mower.context_contract import ContextRequest from code_mower.context_delivery import read_binding from code_mower.context_review import INPUT_HEADER, marker +from code_mower.context_store import ContextStore from code_mower.provider_runners import verdict_artifacts import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -193,3 +201,67 @@ def test_context_bound_artifact_cannot_be_reposted_without_fresh_audit(self): with self.assertRaisesRegex(ValueError, 'fresh authorized audit'): verdict_artifacts.repost_audit_verdict_artifact(result.verdict_artifact_path, token='test-authorization') post.assert_not_called() + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphIndependentReviewTargetTests(unittest.TestCase): + """Independent review replay binds to the immutable audited head, never to + ``repo_path`` or a control checkout (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.head = self.manifest.commit + recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': recipients, + }) + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + result = context_packets.fetch(self.store, 'local-graph', spec, revision=self.head) + # The fixture's own actual revision -- this checkout is genuinely at + # ``self.head`` when the packet is attached -- never the immutable + # review-target head that ``context_audit`` is separately exercised + # against below. + self.current = context_delivery.attach(self.store, 'local-graph', result['packet_handle'], + self.policy, ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, + publish=lambda metadata: None, consuming_revision=self.head) + self.comments = [{'user': {'login': 'controller'}, 'body': INPUT_HEADER + '\n\n' + marker(self.current)}] + self.root = root + + def prepare(self, repo_path): + return context_audit.prepare( + repository='owner/repo', pr=42, head=self.head, host='codex', authorities=('controller',), + fetch_comments=lambda: self.comments, store=self.store, repo_path=repo_path, + ) + + def test_prepare_authorizes_against_the_review_target_head_not_a_foreign_control_checkout(self): + outside = self.root / 'unrelated-control-checkout' + outside.mkdir() + state = self.prepare(outside) + self.assertTrue(state.ready) + self.assertIn('Private evidence', state.text) + + def test_finish_verifies_against_the_review_target_head_not_a_foreign_control_checkout(self): + outside = self.root / 'another-unrelated-control-checkout' + outside.mkdir() + state = self.prepare(outside) + self.assertTrue(state.ready) + prose = 'Summary: preserved the routing constraint. No blocking regressions found.' + self.assertTrue(state.finish(head=self.head, prose=prose)) + self.assertEqual( + read_binding(self.store, self.current['revision'])['feedback']['codex'], prose, + ) diff --git a/tests/test_context_command.py b/tests/test_context_command.py index 43502b55..4b5087a2 100644 --- a/tests/test_context_command.py +++ b/tests/test_context_command.py @@ -11,10 +11,16 @@ from unittest import mock from code_mower import context_command as command, context_packets, work_orders +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_contract import ContextError from code_mower.context_delivery import read_binding from code_mower.context_review import INPUT_HEADER, marker, parse +from code_mower.context_store import ContextStore import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault +from test_context_graph_connection import POLICY as GRAPH_POLICY, RECIPIENTS as GRAPH_RECIPIENTS @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -82,6 +88,15 @@ def test_attach_publishes_pending_before_content_free_input_and_delivers_same_pa self.assertEqual(text, texts[0]) self.assertEqual(self.fixture.backend.searches, 1) + def test_attach_succeeds_for_organization_evidence_from_a_non_git_directory(self): + """Organization evidence never depends on a code revision (codex:65a17212478a56416b1c).""" + outside = tempfile.TemporaryDirectory() + self.addCleanup(outside.cleanup) + code, out, err, _ = self.invoke(['attach', '--connection', 'example', '--host', 'codex', + '--request-stdin', '--repo-path', outside.name], self.attach_spec) + self.assertEqual(code, 0, err) + self.assertIn('"status": "attached"', out) + def test_explicit_unavailable_input_requires_fresh_review_and_never_reads_provider(self): from code_mower import audit_labeler_lib spec = {key: value for key, value in self.attach_spec.items() if key != 'packet'} @@ -165,6 +180,128 @@ def test_revision_cannot_silently_ignore_a_different_explicit_connection(self): self.assertEqual((code, out, credentials), (1, '', 0)) +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphContextCommandTests(unittest.TestCase): + """Standalone attached replay must bind repository evidence to the actual + consuming checkout named by ``--repo-path``, never to the remote PR head + alone (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': GRAPH_RECIPIENTS, + }) + self.head = self.manifest.commit + self.comments = [] + + def invoke(self, args, spec=None, *, actor='controller', remote_head=None): + def github(method, path, **kwargs): + if path == '/user': + return {'login': actor} + return {} + def publish(repo, pr, body, **kwargs): + self.comments.append({'user': {'login': actor}, 'body': body}) + return {'id': 123} + with ExitStack() as stack: + stdout = stack.enter_context(redirect_stdout(io.StringIO())) + stderr = stack.enter_context(redirect_stderr(io.StringIO())) + stack.enter_context(mock.patch.object(command, 'ContextStore', return_value=self.store)) + stack.enter_context(mock.patch.object(command, '_decision_authorities_for_repo', return_value=('controller',))) + stack.enter_context(mock.patch.object(command, '_github', return_value='test-authorization')) + stack.enter_context(mock.patch.object(command, '_gh_request', side_effect=github)) + stack.enter_context(mock.patch.object(command, 'post_pr_comment', side_effect=publish)) + stack.enter_context(mock.patch.object(command, 'fetch_pull_request', + return_value={'head': {'sha': remote_head if remote_head is not None else self.head}})) + stack.enter_context(mock.patch.object(command, 'fetch_issue_comments', return_value=self.comments)) + stack.enter_context(mock.patch.object(command.sys, 'stdin', SimpleNamespace(buffer=io.BytesIO(json.dumps(spec).encode())))) + code = work_orders.context_main(args) + return code, stdout.getvalue(), stderr.getvalue() + + def _attach_spec(self, pr=7): + packet_spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': GRAPH_POLICY} + result = context_packets.fetch(self.store, 'local-graph', packet_spec, revision=self.head) + return {'repository': 'owner/repo', 'work_item': 'WORK-1', 'pr': pr, 'policy': GRAPH_POLICY, + 'packet': result['packet_handle']} + + def attach(self, *, repo_path=None, remote_head=None): + spec = self._attach_spec() + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(repo_path if repo_path is not None else self.repository)], + spec, remote_head=remote_head) + self.assertEqual(code, 0, err) + return json.loads(out)['revision'] + + def test_attachment_succeeds_when_the_checkout_matches_the_pr_head(self): + revision = self.attach() + self.assertTrue(revision) + self.assertEqual(len(self.comments), 1) + + def test_attachment_refuses_a_checkout_that_has_moved_before_publication(self): + """Checkout B must not enable a binding for a PR still reporting head A.""" + spec = self._attach_spec() + graph_fixtures.git(self.repository, 'commit', '--allow-empty', '-q', '-m', 'checkout moved to B') + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(self.repository)], spec) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + + def test_attachment_refuses_a_non_git_directory_before_publication(self): + spec = self._attach_spec() + outside = self.repository.parent / 'not-a-checkout' + outside.mkdir() + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(outside)], spec) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packet_even_when_the_checkout_matches(self): + """Packet A versus PR B is refused even though the consuming checkout is A.""" + spec = self._attach_spec() + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(self.repository)], spec, remote_head='b' * 40) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + + def test_replay_succeeds_from_the_actual_consuming_checkout(self): + revision = self.attach() + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(self.repository)]) + self.assertEqual(code, 0, err) + self.assertIn('Private evidence', out) + + def test_replay_refuses_a_checkout_that_moved_off_the_attachment_head(self): + """Checkout B must not receive evidence bound to checkout A's head.""" + revision = self.attach() + graph_fixtures.git(self.repository, 'commit', '--allow-empty', '-q', '-m', 'checkout moved to B') + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(self.repository)]) + self.assertEqual((code, out), (1, '')) + + def test_replay_refuses_a_non_git_directory(self): + revision = self.attach() + outside = self.repository.parent / 'not-a-checkout' + outside.mkdir() + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(outside)]) + self.assertEqual((code, out), (1, '')) + + class ContextWorkOrderTests(unittest.TestCase): def test_work_order_tracks_only_opaque_packet_and_does_not_expand_cloud_metadata(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_context_delivery.py b/tests/test_context_delivery.py index 2fa6d446..9d4a4ee6 100644 --- a/tests/test_context_delivery.py +++ b/tests/test_context_delivery.py @@ -7,15 +7,19 @@ import unittest from pathlib import Path +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_connections import connect, disconnect from code_mower.context_contract import ContextError, ContextRequest, load_packet -from code_mower.context_delivery import (SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, attach, deliver, public_verdict, - read_binding, render_evidence, save_feedback) -from code_mower.context_packets import fetch +from code_mower.context_delivery import (SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, abandon_attachment, attach, + deliver, public_verdict, read_binding, render_evidence, + retire_attachment, save_feedback) +from code_mower.context_packets import _index, fetch from code_mower.context_store import ContextStore from test_context_connections import MemoryVault from test_context_packets import RetrievalBackend from test_coworker_retrieval import POLICY +import test_context_graph_query as graph_fixtures @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -113,6 +117,11 @@ def test_retrieval_refresh_deletes_old_binding_and_feedback(self): self.delivery(current) self.assertEqual(list(self.store.root.glob('.d-*.json')), []) + def test_organization_replay_succeeds_without_a_consuming_revision(self): + """Organization evidence has no code revision (codex:b7f5dbb1412eb89a3797).""" + current = self.attach() + self.assertTrue(self.delivery(current, consuming_revision=None).text) + def test_public_verdict_has_only_metadata_and_never_model_authored_findings(self): current = self.attach() delivery = self.delivery(current) @@ -162,3 +171,177 @@ def test_local_repository_graph_uses_common_renderer_without_oauth_identity(self self.assertIn('Synthetic evidence: parser calls validator.', texts[0]) self.assertNotIn('/example/repository', texts[0]) self.assertNotIn('principal', texts[0]) + + def test_retirement_completes_after_an_interrupted_index_write(self): + """Delivery removal writes the index without the revision before + deleting the artifact; a crash between those writes must resolve on + retry rather than report an inconsistent index (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + index_file, index = _index(locked) + entry = next(item for item in index['entries'] if item['handle'] == handle) + entry['deliveries'].remove(revision) + index_file.write(index) + # The artifact is still present; only the index write completed. + retire_attachment(self.store, 'example', handle, revision) + with self.assertRaises(ContextError): + read_binding(self.store, revision) + # Retrying the exact same removal is idempotent. + retire_attachment(self.store, 'example', handle, revision) + + def test_abandon_still_refuses_a_published_binding(self): + current = self.attach() + with self.assertRaises(ContextError): + abandon_attachment(self.store, 'example', self.result['packet_handle'], current['revision']) + self.assertTrue(read_binding(self.store, current['revision'])['published']) + + def test_removal_refuses_a_binding_whose_identity_does_not_match(self): + """The targeted binding is identity checked; a mismatched handle or + revision inside the stored binding must still fail closed even + though the artifact exists (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + artifact = locked.artifact('d-' + revision) + corrupted = {**artifact.read(), 'handle': 'f' * 32} + artifact.write(corrupted) + with self.assertRaises(ContextError): + retire_attachment(self.store, 'example', handle, revision) + with self.assertRaises(ContextError): + abandon_attachment(self.store, 'example', handle, revision) + + def test_removal_refuses_when_the_index_entry_is_missing(self): + """A missing index entry is not the same state an interrupted + cleanup leaves -- that only ever drops the revision from an + existing entry's deliveries -- so it still fails closed rather than + completing as if it were (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + index_file, index = _index(locked) + index['entries'] = [item for item in index['entries'] if item['handle'] != handle] + index_file.write(index) + with self.assertRaises(ContextError): + retire_attachment(self.store, 'example', handle, revision) + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphDeliveryBindingTests(unittest.TestCase): + """A repository replay must bind to its explicit consuming revision, never the + attachment/PR head it happens to have been published for (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': self.recipients, + }) + self.head = self.manifest.commit # checkout A + self.other = 'b' * 40 # a different checkout, B + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + result = fetch(self.store, 'local-graph', spec, revision=self.head) + self.current = attach(self.store, 'local-graph', result['packet_handle'], self.policy, + ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, + publish=lambda metadata: None, consuming_revision=self.head) + + def delivery(self, **kwargs): + return deliver(self.store, self.current['revision'], repository='owner/repo', pr=42, head=self.head, + recipient='codex:builder', current=self.current, **kwargs) + + def test_replay_matches_the_actual_consuming_revision(self): + self.assertIn('Private evidence', self.delivery(consuming_revision=self.head).text) + + def test_replay_refuses_a_different_consuming_revision(self): + """Checkout B must not receive evidence authorized for checkout A.""" + with self.assertRaises(ContextError): + self.delivery(consuming_revision=self.other) + + def test_replay_never_silently_substitutes_the_attachment_head(self): + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + with self.assertRaises(ContextError): + self.delivery() + with self.assertRaises(ContextError): + self.delivery(consuming_revision=None) + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphAttachmentBindingTests(unittest.TestCase): + """A fresh attachment must bind repository evidence to the actual consuming + checkout, never merely to the PR head a caller happens to report + (codex:65a17212478a56416b1c).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': self.recipients, + }) + self.head = self.manifest.commit # checkout A + self.other = 'b' * 40 # a different checkout, B + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + self.handle = fetch(self.store, 'local-graph', spec, revision=self.head)['packet_handle'] + self.published = [] + + def attach(self, *, head, **kwargs): + return attach(self.store, 'local-graph', self.handle, self.policy, + ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=head, + publish=self.published.append, **kwargs) + + def test_attachment_succeeds_when_packet_pr_and_consumer_all_match(self): + current = self.attach(head=self.head, consuming_revision=self.head) + self.assertEqual(current['head'], self.head) + self.assertEqual(len(self.published), 1) + + def test_attachment_refuses_a_consumer_the_checkout_is_not_actually_on(self): + """Checkout B must not enable a binding authorized for checkout A.""" + with self.assertRaises(ContextError): + self.attach(head=self.head, consuming_revision=self.other) + self.assertEqual(self.published, []) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packet_even_when_the_consumer_matches(self): + """Packet A versus PR B is refused even though the consumer is A.""" + with self.assertRaises(ContextError): + self.attach(head=self.other, consuming_revision=self.head) + self.assertEqual(self.published, []) + + def test_attachment_never_silently_substitutes_the_pr_head_for_an_unknown_consumer(self): + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + with self.assertRaises(ContextError): + self.attach(head=self.head) + with self.assertRaises(ContextError): + self.attach(head=self.head, consuming_revision=None) + self.assertEqual(self.published, []) diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py new file mode 100644 index 00000000..f1805b2e --- /dev/null +++ b/tests/test_context_graph_connection.py @@ -0,0 +1,721 @@ +"""The guided route over a local repository graph (issue #914). + +These tests drive the ordinary guided path -- ``context_prepare.prepare``, the +shared packet store, and ``context_delivery`` -- against a Graphify-kind local +connection rather than an organization one. What they are here to prove is not +the packet format, which ``test_context_graph_query`` already covers, but that +the guided route reaches it: that preparation mints a packet through the shared +store, that reuse does not re-query, that Claude, Codex and Devin receive +byte-identical approved evidence, and that a graph which was rebuilt or whose +revision has moved on is refused rather than replayed. + +No Coworker SDK, credential, or network call takes part. The graph document is +written by an injected indexer, as in the query tests, so nothing here claims a +provider was installed. +""" + +from __future__ import annotations + +import io +import json +import os +import tempfile +import unittest +from contextlib import chdir, redirect_stdout +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from code_mower import context_delivery, context_packets, context_prepare, context_session +from code_mower import context_graph_connection as connection +from code_mower import context_graph_lifecycle as lifecycle +from code_mower import context_graph_query as query +from code_mower.context_contract import ContextError, ContextRequest +from code_mower.context_store import ContextStore +from test_context_connections import MemoryVault +from test_context_graph_query import ( + PIN, git, graph_document, indexer, make_repository, wide_graph_document, +) + + +POLICY = { + "schema": "code_mower.contextPolicy.v1", + "connection": "local-graph", + "policy_version": "v1", + "required": True, +} +RECIPIENTS = [ + "codex:orchestrator", "claude:orchestrator", "devin:orchestrator", + "codex:builder", "claude:builder", "devin:builder", + "codex:reviewer", "claude:reviewer", "devin:reviewer", +] + + +def session_value(session_id: str = "a" * 32, *, host: str = "codex") -> dict: + return { + "schema": "code_mower.session.v1", + "id": session_id, + "repo": "owner/repo", + "host": host, + "orchestrator": host, + "participants": [{"id": "claude"}, {"id": "codex"}], + "lease": {"state": "held", "mutating": True}, + } + + +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class GuidedGraphSessionTests(unittest.TestCase): + """One checkout, one published generation, one guided session over it.""" + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = make_repository(self.root) + # One private root for both the graph's generations and the packet + # store, which is the arrangement an operator actually gets: the + # lifecycle and the context store share ``default_context_root``. + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = self.publish() + self.store = ContextStore(self.private, vault=MemoryVault()) + self.associations = context_session.association_store(self.private) + connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + }) + + def publish(self, document: dict | None = None): + return lifecycle.build_graph( + self.repository, + pin=PIN, + indexer=indexer(document or graph_document()), + root=self.private, + ) + + def spec(self, *, question: str = "impact", target: str = "parse_config", + recipient: str = "codex:orchestrator", required: bool = True) -> dict: + return { + "repository": "owner/repo", "work_item": "WORK-1", "recipient": recipient, + "query": target, "source": question, + "policy": {**POLICY, "required": required}, + } + + def fetch(self, *, revision: str = "HEAD", **overrides): + return context_packets.fetch( + self.store, "local-graph", self.spec(**overrides), revision=revision, + ) + + def load(self, handle: str, recipient: str, revision: str = "HEAD"): + return context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", recipient, revision), + ) + + # -- retrieval through the shared store ------------------------------ + + def test_retrieval_binds_the_published_generation_and_the_graphs_commit(self) -> None: + result = self.fetch() + self.assertEqual((result["status"], result["reused"]), ("available", False)) + # No paid-provider usage exists for a local graph, and reporting a + # fabricated zero would read as "a search happened and cost nothing". + self.assertIsNone(result["usage"]) + packet = self.load(result["packet_handle"], "claude:builder").private_payload() + self.assertEqual(packet["provider"], connection.PROVIDER) + self.assertEqual(packet["kind"], "repository") + self.assertEqual(packet["source_revision"], self.manifest.commit) + self.assertEqual(packet["binding"]["generation"], self.manifest.generation) + self.assertEqual( + packet["binding"]["identity"], {"repository_root": str(self.repository)}, + ) + + def test_reuse_returns_the_same_packet_without_traversing_again(self) -> None: + first = self.fetch() + again = self.fetch() + self.assertEqual(again["packet_handle"], first["packet_handle"]) + self.assertTrue(again["reused"]) + + def test_a_different_question_is_a_different_packet(self) -> None: + impact = self.fetch(question="impact") + symbol = self.fetch(question="symbol") + self.assertNotEqual(symbol["packet_handle"], impact["packet_handle"]) + self.assertFalse(symbol["reused"]) + + def test_an_unsupported_question_is_refused_before_any_traversal(self) -> None: + with self.assertRaises(ContextError): + self.fetch(question="everything") + + def test_an_unapproved_recipient_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.fetch(recipient="cursor:builder") + + # -- identical approved evidence ------------------------------------- + + def test_claude_codex_and_devin_receive_identical_approved_evidence(self) -> None: + handle = self.fetch()["packet_handle"] + rendered = { + host: context_delivery.render_evidence( + self.load(handle, f"{host}:builder"), handle, + ) + for host in context_delivery.SUPPORTED_HOSTS + } + self.assertEqual(set(rendered), {"claude", "codex", "devin"}) + self.assertEqual(len(set(rendered.values())), 1) + evidence = rendered["claude"] + self.assertIn("example_pkg/config.py#L12", evidence) + # The recipient needs no provider, pin, or graph tool to read this: the + # evidence names the provider, as it does for an organization packet, + # and carries no private path, checkout root, or connection identity. + for private in (str(self.private), str(self.repository), "local-graph"): + self.assertNotIn(private, evidence) + + # -- freshness: rebuilt, moved, or absent ---------------------------- + + def test_a_rebuilt_graph_refuses_the_packet_bound_to_the_old_generation(self) -> None: + handle = self.fetch()["packet_handle"] + republished = self.publish() + self.assertNotEqual(republished.generation, self.manifest.generation) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_a_moved_head_refuses_delivery_rather_than_answering_for_the_old_commit(self) -> None: + handle = self.fetch()["packet_handle"] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_an_explicitly_named_prior_revision_still_loads_its_own_packet(self) -> None: + """Freshness is about the requested revision, not about wall-clock time.""" + handle = self.fetch()["packet_handle"] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + packet = context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder"), + revision=self.manifest.commit, + ) + self.assertEqual(packet.private_payload()["source_revision"], self.manifest.commit) + + def _second_commit_the_checkout_is_not_on(self) -> str: + """Make a commit, then leave the registered checkout back on the first. + + This is the arrangement the consuming-revision rule exists for: the + connected checkout still sits at the commit its graph was built from, + while the work consuming the evidence is at another commit entirely -- + a builder's branch, a PR head, a worktree. Both commits are real and + resolvable, so nothing here fails for want of an object. + """ + first = lifecycle.resolve_revision(self.repository)[0] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "consuming work") + second = lifecycle.resolve_revision(self.repository)[0] + git(self.repository, "reset", "-q", "--hard", first) + self.assertEqual(lifecycle.resolve_revision(self.repository)[0], self.manifest.commit) + self.assertNotEqual(second, self.manifest.commit) + return second + + def test_retrieval_refuses_a_graph_that_is_not_the_consuming_revisions(self) -> None: + """The registered checkout's ``HEAD`` is not the revision being worked on.""" + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self.fetch(revision=consuming) + + def test_replay_refuses_a_packet_for_another_revisions_code(self) -> None: + handle = self.fetch()["packet_handle"] + consuming = self._second_commit_the_checkout_is_not_on() + # The checkout's own HEAD still authorizes, which is exactly why the + # consuming revision has to be the one asked about. + self.assertEqual(self.load(handle, "claude:builder").revision_state, "matching") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder", revision=consuming) + + def test_a_load_that_cannot_name_its_consuming_revision_is_refused(self) -> None: + handle = self.fetch()["packet_handle"] + with self.assertRaises(ContextError): + context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder"), + ) + + def test_removing_the_graph_makes_the_connection_unavailable(self) -> None: + handle = self.fetch()["packet_handle"] + lifecycle.remove_graph(self.repository, root=self.private) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_disconnecting_disables_the_connection_and_drops_its_packets(self) -> None: + handle = self.fetch()["packet_handle"] + summary = connection.disconnect(self.store, "local-graph") + self.assertEqual((summary["status"], summary["packet_cleanup"]), ("disconnected", "complete")) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def _reconnect(self, **overrides) -> dict: + spec = { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + } + spec.update(overrides) + return connection.connect(self.store, "local-graph", spec) + + def test_reconnect_completes_pending_packet_cleanup_that_disconnect_could_not(self) -> None: + """A cleanup failure on disconnect must not let reconnect skip it. + + Reproduces codex:57555c67a42d8aeba514: previously ``connect`` wrote + ``verified`` state straight back once a disconnected connection was + found, regardless of whether the packets it once authorized were ever + actually purged. With the same graph and approved scope, that let a + surviving packet and its delivery/attachment binding become authorized + again. + """ + handle = self.fetch()["packet_handle"] + metadata = self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) + current = context_delivery.mark_published(self.store, "local-graph", metadata["revision"]) + with patch("code_mower.context_packets.purge_connection", side_effect=RuntimeError("boom")): + summary = connection.disconnect(self.store, "local-graph") + self.assertEqual((summary["status"], summary["packet_cleanup"]), ("disconnected", "needs_attention")) + with self.assertRaises(ContextError): + self._reconnect() + # Cleanup is still pending: the connection stays disconnected and the + # surviving packet cannot be loaded. ``read_binding`` is only a + # private file lookup with no authorization of its own, so the + # binding it names is expected to still be there -- that survival is + # exactly the pending cleanup, not a security property by itself. + with self.store.locked("local-graph") as locked: + self.assertEqual(connection.saved_state(locked.read(), "local-graph")["state"], "disconnected") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + survived = context_delivery.read_binding(self.store, metadata["revision"]) + self.assertTrue(survived["published"]) + # The actual security property: a published binding must not replay + # while the connection that authorized it is disconnected. + with self.assertRaises(ContextError): + context_delivery.deliver( + self.store, metadata["revision"], repository="owner/repo", pr=1, head=self.manifest.commit, + recipient="claude:reviewer", current=current, consuming_revision=self.manifest.commit, + ) + # Cleanup now succeeds: reconnect completes it and only then verifies. + self.assertEqual(self._reconnect()["status"], "verified") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + with self.assertRaises(ContextError): + context_delivery.read_binding(self.store, metadata["revision"]) + + def test_connection_status_reports_the_graph_without_minting_evidence(self) -> None: + report = connection.status(self.store, "local-graph", root=self.private) + self.assertEqual(report["provider"], connection.PROVIDER) + self.assertEqual(report["authorization"], "available") + self.assertTrue(report["graph"]["usable"]) + + def test_authorization_names_a_repository_kind_without_an_account(self) -> None: + with self.store.locked("local-graph") as locked: + envelope = connection.authorize_locked( + locked, "local-graph", root=self.private, now=datetime.now(timezone.utc), + ) + self.assertEqual(envelope["kind"], "repository") + self.assertEqual(envelope["generation"], self.manifest.generation) + self.assertEqual( + envelope["capabilities"], + {"search": True, "memory": False, "revision_binding": True}, + ) + self.assertNotIn("principal", envelope["identity"]) + + # -- the guided session verbs ---------------------------------------- + + def record(self, *, host: str = "codex", required: bool = True): + return context_session.create( + self.associations, + session_value(host=host), + work_item="WORK-1", + policy={**POLICY, "required": required}, + ) + + def prepare(self, record, **kwargs): + arguments = {"query": "parse_config", "source": "impact"} + arguments.update(kwargs) + return context_prepare.prepare( + self.associations, + record, + repo_root=self.repository, + context_root=self.private, + packet_store=self.store, + **arguments, + ) + + def test_prepare_then_reuse_produces_one_packet_the_builder_can_read(self) -> None: + record = self.record() + first, code = self.prepare(record) + self.assertEqual((code, first["status"], first["reused"]), (0, "prepared", False)) + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual(saved["stage"], "prepared") + self.assertIsNotNone(saved["packet"]) + + again, code = self.prepare(saved) + self.assertEqual((code, again["status"], again["reused"]), (0, "prepared", True)) + self.assertEqual( + context_session.read(self.associations, record["session_id"])["packet"], + saved["packet"], + ) + evidence = context_delivery.render_evidence( + self.load(saved["packet"], "codex:builder"), saved["packet"], + ) + self.assertIn("example_pkg/config.py#L12", evidence) + + def test_prepare_pauses_required_work_when_the_graph_is_stale(self) -> None: + record = self.record(required=True) + lifecycle.remove_graph(self.repository, root=self.private) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + self.assertEqual(report["dependent_work"], "paused") + + def test_prepare_degrades_optional_work_when_the_graph_is_stale(self) -> None: + record = self.record(required=False) + lifecycle.remove_graph(self.repository, root=self.private) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + self.assertEqual(report["dependent_work"], "usable") + + def test_reuse_after_a_rebuild_pauses_required_work(self) -> None: + record = self.record(required=True) + self.prepare(record) + saved = context_session.read(self.associations, record["session_id"]) + self.publish() + report, code = self.prepare(saved) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + + # -- a wide answer under the default document budget ------------------- + + def test_a_wide_answer_is_prepared_bounded_rather_than_refused_at_delivery(self) -> None: + """Six citable relationships, a budget of five, one usable packet. + + The default policy carries no ``max_documents``, so the contract's own + default of five applies -- and the contract enforces it when the packet + is loaded. A packet built to the query adapter's ceiling instead would + pass preparation and then fail every authorized load, which is a + required session paused over evidence the graph actually had. What the + builder must get is the bounded answer, told plainly that it is bounded. + """ + self.manifest = self.publish(wide_graph_document(6)) + record = self.record() + report, code = self.prepare(record) + self.assertEqual((code, report["status"], report["dependent_work"]), (0, "prepared", "usable")) + + saved = context_session.read(self.associations, record["session_id"]) + loaded = self.load(saved["packet"], "claude:builder") + packet = loaded.private_payload() + self.assertEqual(len(packet["documents"]), 5) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + self.assertEqual(packet["binding"]["generation"], self.manifest.generation) + # The builder reads it as evidence, not as a handle it cannot open. + evidence = context_delivery.render_evidence(loaded, saved["packet"]) + self.assertIn("example_pkg/config.py#L12", evidence) + + def test_replaying_a_bounded_packet_keeps_its_omissions_and_binding(self) -> None: + self.manifest = self.publish(wide_graph_document(6)) + record = self.record() + self.prepare(record) + saved = context_session.read(self.associations, record["session_id"]) + again, code = self.prepare(saved) + self.assertEqual((code, again["status"], again["reused"]), (0, "prepared", True)) + self.assertEqual( + context_session.read(self.associations, record["session_id"])["packet"], + saved["packet"], + ) + first = self.load(saved["packet"], "claude:builder").private_payload() + replayed = self.load(saved["packet"], "codex:builder").private_payload() + self.assertEqual(replayed["omissions"], first["omissions"]) + self.assertEqual(replayed["truncated"], first["truncated"]) + self.assertEqual(replayed["binding"], first["binding"]) + self.assertEqual(len(replayed["documents"]), 5) + + def test_an_optional_wide_answer_is_delivered_rather_than_degraded(self) -> None: + """A budget is not unavailability: optional work gets the evidence too.""" + self.manifest = self.publish(wide_graph_document(6)) + record = self.record(required=False) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (0, "prepared")) + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual( + len(self.load(saved["packet"], "claude:builder").private_payload()["documents"]), 5, + ) + + def _attach(self, handle: str, head: str, *, consuming_revision=None): + return context_delivery.reserve_attachment( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "codex:orchestrator"), + pr=1, head=head, consuming_revision=consuming_revision, + ) + + def test_attachment_refuses_a_packet_whose_graph_was_rebuilt(self) -> None: + handle = self.fetch()["packet_handle"] + self.publish() + with self.assertRaises(ContextError): + self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) + + def test_attachment_succeeds_when_packet_pr_and_consumer_all_match(self) -> None: + handle = self.fetch()["packet_handle"] + # The head the graph *is* for, and the checkout it is actually on, both attach. + self.assertEqual( + self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit)["head"], + self.manifest.commit, + ) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packets_revision(self) -> None: + """Packet A versus PR B is refused even though the consumer is A.""" + handle = self.fetch()["packet_handle"] + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self._attach(handle, consuming, consuming_revision=self.manifest.commit) + + def test_attachment_refuses_a_consumer_the_checkout_is_not_actually_on(self) -> None: + """A checkout at another commit must not enable a binding for this PR head.""" + handle = self.fetch()["packet_handle"] + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self._attach(handle, self.manifest.commit, consuming_revision=consuming) + + def test_attachment_never_substitutes_the_pr_head_for_an_unknown_consumer(self) -> None: + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + handle = self.fetch()["packet_handle"] + with self.assertRaises(ContextError): + self._attach(handle, self.manifest.commit) + + +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class StandaloneGraphFetchCommandTests(unittest.TestCase): + """``code-mower context fetch`` run outside a guided session (issue #914). + + The guided route reads the consuming checkout's revision itself, from the + directory the session is preparing work in. The standalone command has no + session record to read it from, so what is proven here is that it reads that + revision from the checkout it was *run in*, rather than leaving ``fetch`` to + resolve the word ``HEAD`` in whichever checkout the connection happens to + have been registered against. Those are two directories that move + independently, and the second reading is exactly how evidence describing one + commit's code reaches work on another. + + The real entrypoint runs. Only the store root is injected -- the command + otherwise builds its own -- so the packet these tests read back came through + the same protected file, the same index, and the same authorization a + guided delivery goes through. + """ + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = make_repository(self.root) + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=PIN, indexer=indexer(graph_document()), root=self.private, + ) + self.store = ContextStore(self.private, vault=MemoryVault()) + connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + }) + + def run_command(self, cwd: Path, *, required: bool = True) -> tuple[int, dict]: + """Drive the real command from ``cwd``. No provider backend may load.""" + spec = { + "repository": "owner/repo", "work_item": "WORK-1", "recipient": "claude:builder", + "query": "parse_config", "source": "impact", + "policy": {**POLICY, "required": required}, + } + output = io.StringIO() + with patch("code_mower.context_packets.ContextStore", return_value=self.store), \ + patch("code_mower.context_packets._backend", + side_effect=AssertionError("a local graph must not reach the provider SDK")), \ + patch("sys.stdin", SimpleNamespace(buffer=io.BytesIO(json.dumps(spec).encode()))), \ + chdir(cwd), redirect_stdout(output): + code = context_packets.main(["--connection", "local-graph", "--request-stdin", "--json"]) + return code, json.loads(output.getvalue()) + + def _another_checkout_at_a_commit_this_one_is_not_on(self) -> Path: + """A second checkout at a commit the registered one has moved off. + + The clone is taken while the second commit is current, so that checkout + keeps it; the registered checkout is then put back on the commit its + graph was built from. The object stays reachable there, so the refusal + under test is a real comparison of two resolvable commits rather than a + name the graph's repository could not look up at all. + """ + first = lifecycle.resolve_revision(self.repository)[0] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "consuming work") + second = lifecycle.resolve_revision(self.repository)[0] + consuming = self.root / "consuming" + git(self.root, "clone", "-q", str(self.repository), str(consuming)) + git(self.repository, "reset", "-q", "--hard", first) + self.assertEqual(lifecycle.resolve_revision(self.repository)[0], self.manifest.commit) + self.assertEqual(lifecycle.resolve_revision(consuming)[0], second) + self.assertNotEqual(second, self.manifest.commit) + return consuming + + def test_running_in_the_graphs_own_checkout_delivers_its_packet(self) -> None: + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + packet = context_packets.load_authorized( + self.store, "local-graph", report["packet_handle"], POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder", self.manifest.commit), + ) + self.assertEqual(packet.private_payload()["source_revision"], self.manifest.commit) + self.assertEqual(packet.private_payload()["binding"]["generation"], self.manifest.generation) + + def test_a_consumer_at_another_revision_is_refused_rather_than_answered(self) -> None: + consuming = self._another_checkout_at_a_commit_this_one_is_not_on() + code, report = self.run_command(consuming) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + # Refused at authorization, before anything is reserved: no packet of + # the wrong commit's evidence exists to be replayed later. + self.assertEqual(list(self.private.glob(".p-*.json")), []) + # And the discrimination is the consuming revision alone. The registered + # checkout still sits at the graph's commit, so a command that resolved + # ``HEAD`` there would have answered the request above as it answers + # this one. + self.assertEqual(self.run_command(self.repository)[1]["status"], "available") + + def test_an_optional_consumer_at_another_revision_degrades_instead_of_pausing(self) -> None: + consuming = self._another_checkout_at_a_commit_this_one_is_not_on() + code, report = self.run_command(consuming, required=False) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + self.assertEqual(list(self.private.glob(".p-*.json")), []) + + def _publish_wide(self, callers: int = 6): + """Republish this checkout's graph with ``callers`` citable relationships.""" + self.manifest = lifecycle.build_graph( + self.repository, pin=PIN, indexer=indexer(wide_graph_document(callers)), + root=self.private, + ) + return self.manifest + + def test_a_wide_answer_is_delivered_bounded_instead_of_failing_validation(self) -> None: + """Six relationships, the contract's default budget of five, one packet. + + Before the document budget reached the traversal, the command built six + documents and the shared contract refused them on the way into the + store: ``packet_invalid``, required work paused, over evidence the graph + had and the policy simply did not have room for. The answer is five + documents that say they are five of more, and no validation failure. + """ + manifest = self._publish_wide(6) + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertNotEqual(report.get("reason"), "packet_invalid") + self.assertNotIn("failure_reason", report) + self.assertEqual(report["documents"], 5) + self.assertEqual(report["completeness"], "partial") + self.assertTrue(report["truncated"]) + + packet = context_packets.load_authorized( + self.store, "local-graph", report["packet_handle"], POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder", manifest.commit), + ).private_payload() + self.assertEqual(len(packet["documents"]), 5) + self.assertIn("document_limit", packet["omissions"]) + self.assertEqual(packet["binding"]["generation"], manifest.generation) + # Every delivered citation still points at the immutable tree. + self.assertTrue(all( + citation["source"].startswith(("example_pkg/", "tests/")) + for item in packet["documents"] for citation in item["citations"] + )) + + def test_an_optional_wide_answer_is_delivered_rather_than_degraded(self) -> None: + """A budget is not unavailability: the optional caller gets evidence too.""" + self._publish_wide(6) + code, report = self.run_command(self.repository, required=False) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertNotEqual(report.get("reason"), "packet_invalid") + self.assertEqual(report["documents"], 5) + self.assertTrue(report["truncated"]) + + def test_an_answer_that_fits_the_budget_exactly_is_still_complete(self) -> None: + """The bound is only reported when it actually left evidence out.""" + self._publish_wide(5) + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertEqual(report["documents"], 5) + self.assertEqual(report["completeness"], "complete") + self.assertFalse(report["truncated"]) + + def test_a_consumer_that_is_not_a_checkout_is_refused_rather_than_defaulted(self) -> None: + """No revision at all is a refusal, not a fall back to the graph's.""" + elsewhere = self.root / "not-a-checkout" + elsewhere.mkdir() + code, report = self.run_command(elsewhere) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + self.assertEqual(list(self.private.glob(".p-*.json")), []) + code, report = self.run_command(elsewhere, required=False) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + + +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class GraphConnectionStateTests(unittest.TestCase): + """What the saved connection will and will not accept.""" + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.repository = make_repository(self.root) + self.store = ContextStore(self.private, vault=MemoryVault()) + + def connect(self, **overrides): + spec = { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": ["claude:builder"], + } + spec.update(overrides) + return connection.connect(self.store, "local-graph", spec) + + def test_connecting_stores_no_credential_and_reports_no_vault(self) -> None: + summary = self.connect() + self.assertEqual(summary["credential_storage"], "none") + self.assertEqual(summary["kind"], "repository") + + def test_a_relative_checkout_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.connect(repository_root="checkout") + + def test_reconnecting_a_live_connection_is_refused(self) -> None: + self.connect() + with self.assertRaises(ContextError): + self.connect(repositories=["owner/other"]) + + def test_reconnecting_after_disconnect_is_allowed(self) -> None: + self.connect() + connection.disconnect(self.store, "local-graph") + self.assertEqual(self.connect(repositories=["owner/other"])["status"], "verified") + + def test_the_question_defaults_to_symbol_and_rejects_anything_else(self) -> None: + self.assertEqual( + connection.question_and_target({"query": "parse_config", "source": None}), + (connection.DEFAULT_QUESTION, "parse_config"), + ) + self.assertIn(connection.DEFAULT_QUESTION, query.QUESTIONS) + with self.assertRaises(ContextError): + connection.question_and_target({"query": "parse_config", "source": "anything"}) + + def test_saved_state_refuses_another_providers_connection(self) -> None: + self.assertFalse(connection.is_graph({"schema": "code_mower.contextLocalConnection.v1"})) + with self.assertRaises(ContextError): + connection.saved_state({"schema": connection.GRAPH_SCHEMA}, "local-graph") + + +if __name__ == "__main__": # pragma: no cover - direct invocation + unittest.main() diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index a29a8882..979835be 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -25,11 +25,13 @@ import os import socket import stat +import struct import subprocess import sys import tarfile import tempfile import time +import unicodedata import unittest import uuid from datetime import datetime, timezone @@ -129,14 +131,19 @@ def kill(self) -> None: def recording_indexer(payload: bytes = b"graph-bytes", *, completeness: str = lifecycle.COMPLETE, - seen: list | None = None, indexed_files: int = 0): + seen: list | None = None, indexed_files: int = 0, + unsupported_inputs: int = 0): """An indexer that writes a fixed artifact and records what it was shown.""" def run(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: if seen is not None: seen.append(request) request.output_path.write_bytes(payload) - return lifecycle.IndexResult(completeness=completeness, indexed_files=indexed_files) + return lifecycle.IndexResult( + completeness=completeness, + indexed_files=indexed_files, + unsupported_inputs=unsupported_inputs, + ) return run @@ -1014,9 +1021,71 @@ def classify(prefix, **keywords): self.assertEqual(calls, [()]) -#: What a provider that finished leaves behind: a completion claim and counts -#: that admit nothing outstanding. -FINISHED_REPORT = {"complete": True, "code_files": 3, "requeued": 0} +#: The census a launch fixture indexes: one Python file, one Markdown file. +#: Only the first is a code input to the pin, so only the first is the +#: denominator completeness is measured against. +CODE_INPUT = "src/app.py" +DOC_INPUT = "docs/guide.md" + +#: The bytes a launch fixture actually materializes for each of those, because +#: a manifest row is now checked against them rather than merely shaped like a +#: hash. A census that named files the copy does not hold would be a build that +#: cannot check any row, which is its own (partial) case and has its own test. +FIXTURE_INPUTS = { + CODE_INPUT: b"def main():\n return 0\n", + DOC_INPUT: b"# guide\n", +} + +#: What the pin's ``_md5_file`` would return for the code input above. The +#: adapter derives the same value from the copy before the launch, so this is +#: the one digest a finished manifest can carry for ``src/app.py``. +CODE_INPUT_DIGEST = hashlib.md5(FIXTURE_INPUTS[CODE_INPUT], usedforsecurity=False).hexdigest() + +#: The same rendered filename in its two canonically equivalent spellings: +#: ``é`` as U+00E9, and ``e`` followed by the U+0301 combining acute. Git holds +#: exact bytes, so these are two tracked entries with two blobs, and a Linux +#: checkout carries both at once. They are meant to render identically -- that +#: is the whole point -- so nothing here tells them apart by eye, and a census +#: fixture rather than a real checkout is the platform-independent way to test +#: the pair: macOS cannot hold both names at the same time. +NFC_INPUT = "src/café.py" +NFD_INPUT = "src/café.py" + + +def census_of(*paths: str) -> lifecycle.TrackedCensus: + """A census naming ``paths``, shaped the way ``read_tracked_census`` does.""" + entries = tuple( + lifecycle.TrackedEntry( + path=path, mode="100644", blob="0" * 40, size=len(FIXTURE_INPUTS.get(path, b"x")) + ) + for path in paths + ) + return lifecycle.TrackedCensus(entries=entries, skipped=(), digest="0" * 64) + + +def materialize_fixture_inputs(source_root: Path, census: lifecycle.TrackedCensus) -> None: + """Write the census's own bytes into the copy, as a real build would. + + ``materialize_tracked_files`` puts the commit's blobs here before the + provider is launched; the launch fixtures used to leave the copy empty, + which no longer describes a build whose completeness check reads those + bytes. + """ + for entry in census.entries: + destination = source_root.joinpath(*entry.path.split("/")) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(FIXTURE_INPUTS.get(entry.path, b"x")) + + +def manifest_row(digest: str = CODE_INPUT_DIGEST) -> dict: + """One ``save_manifest`` row, in the pin's own shape.""" + return {"mtime": 1.0, "seen": 2.0, "ast_hash": digest, "semantic_hash": digest} + + +#: What a provider that processed every code input leaves behind: a row per +#: dispatched file, each carrying the content hash that proves its bytes were +#: read. No completion flag and no count -- the pinned manifest has neither. +FINISHED_MANIFEST = {CODE_INPUT: manifest_row()} class ProviderLaunchTests(TemporaryWorkspace): @@ -1053,6 +1122,11 @@ def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: # than exercise anything, so each request names its own. artifact = Path(tempfile.mkdtemp(dir=self.root)) / "graph.bin" self.artifacts.append(artifact) + census = census_of(CODE_INPUT, DOC_INPUT) + # A real build materializes the census into the copy before launching. + # The completeness check now reads those bytes, so the fixture has to + # actually hold them rather than name them. + materialize_fixture_inputs(source, census) return lifecycle.IndexRequest( source_root=source, output_path=artifact, @@ -1060,6 +1134,7 @@ def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: pin=pin, commit="a" * 40, tree="b" * 40, + census=census, ) def run_indexer( @@ -1067,15 +1142,15 @@ def run_indexer( executable: str, *, sandbox=("/sandbox", "--deny"), - report: object = FINISHED_REPORT, - state_directory: str = ".graphify", + report: object = FINISHED_MANIFEST, + state_directory: str = lifecycle._PROVIDER_OUTPUT_DIRECTORY, pin: lifecycle.GraphifyPin = PIN, ) -> tuple[list[str], lifecycle.IndexResult]: """Launch the adapter with the provider's side of the contract faked. - ``extract`` writes its state beside the sources it was run over, so the - stand-in has to leave that state behind for the adapter to collect -- - an exit status alone is not a finished build. + ``extract`` writes ``graphify-out/`` beneath the tree it was run over, + so the stand-in has to leave that output behind for the adapter to + collect -- an exit status alone is not a finished build. """ request = self.request(pin) recorded: list[list[str]] = [] @@ -1089,7 +1164,7 @@ def fake_popen(argv, **kwargs): if state_directory: written = request.source_root / state_directory written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "graph.json").write_text("{}", encoding="utf-8") if report is not None: (written / "manifest.json").write_text(json.dumps(report), encoding="utf-8") return FakeChild() @@ -1132,10 +1207,10 @@ def record_prefix(**keywords: object) -> tuple[str, ...]: return ("/sandbox",) def fake_popen(argv, **kwargs): - written = request.source_root / ".graphify" + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") - (written / "manifest.json").write_text(json.dumps(FINISHED_REPORT), encoding="utf-8") + (written / "graph.json").write_text("{}", encoding="utf-8") + (written / "manifest.json").write_text(json.dumps(FINISHED_MANIFEST), encoding="utf-8") return FakeChild() with stand_in_containment(("/sandbox",)): @@ -1176,13 +1251,68 @@ def test_the_provider_leads_its_own_process_group(self) -> None: def test_the_provider_is_invoked_through_its_documented_extract_interface(self) -> None: # The interface the adopt decision evaluated, recorded in - # docs/graphify-evaluation.md as ``extract`` plus options. An - # ``index --source ... --output ...`` shape would be a different CLI. + # docs/graphify-evaluation.md as ``extract`` over a scan target plus + # options. An ``index --source ... --output ...`` shape would be a + # different CLI. argv = self.launched_argv("graphify") - self.assertEqual(argv[3:], ["extract", *PIN.options]) + self.assertEqual(argv[3:], ["extract", ".", *PIN.options]) self.assertNotIn("--source", argv) self.assertNotIn("--output", argv) + def test_the_scan_target_is_passed_where_the_pinned_cli_reads_it(self) -> None: + """The pinned CLI requires a target and reads it from one position only. + + It takes the target as the first positional after the subcommand, + decides it has one only when that argument does not begin with ``-``, + and exits 1 with ``must specify a path to scan or a --postgres DSN`` + otherwise. The launch used to pass the options alone, so every real + build failed before extraction -- and failed as the adapter's generic + non-zero refusal, which names nothing about the omission. + + ``.`` rather than an absolute path: the child's working directory is + already the materialized copy, so the relative spelling names that tree + and nothing about where it sits on this host. + """ + argv = self.launched_argv("graphify") + subcommand = argv.index("extract") + self.assertEqual(argv[subcommand + 1], ".") + self.assertFalse(argv[subcommand + 1].startswith("-")) + # Before the options, not after them: an argument that follows a flag is + # read as that flag's value or skipped, and either way the CLI still + # sees no path. + for option in PIN.options: + self.assertLess(subcommand + 1, argv.index(option)) + + def test_the_pinned_cli_would_accept_this_argv(self) -> None: + """Read the requirement off the staged pinned source, not off a memory. + + The reference under ``.build/graphify-914-reference`` is the CLI this + pin actually installs. If a later pin stops requiring a positional + target, or starts reading it from somewhere other than ``sys.argv[2]``, + this test is what says so rather than a real build failing opaquely. + """ + reference = ( + Path(__file__).resolve().parent.parent + / ".build" + / "graphify-914-reference" + / "graphify-cli-pinned.py" + ) + if not reference.is_file(): + # The reference is a local read-only staging of the pinned provider's + # own source, excluded from the repository rather than vendored into + # it, so this check runs where it is staged and skips where it is not. + self.skipTest("pinned CLI reference is not staged in this checkout") + source = reference.read_text(encoding="utf-8", errors="replace") + self.assertIn("error: must specify a path to scan or a --postgres DSN", source) + # The target is ``sys.argv[2]`` and a leading dash means "no path". + self.assertIn('if sys.argv[2].startswith("-"):', source) + argv = self.launched_argv("graphify") + provider = argv.index(str(self.provider)) + # ``sys.argv`` inside the child is the provider and everything after it, + # so ``sys.argv[2]`` is the second argument past the executable. + self.assertEqual(argv[provider + 1], "extract") + self.assertEqual(argv[provider + 2], ".") + def test_extraction_is_restricted_to_code_and_never_clusters(self) -> None: """The adoption conditions are enforced at the launch, not assumed. @@ -1193,7 +1323,7 @@ def test_extraction_is_restricted_to_code_and_never_clusters(self) -> None: """ bare = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) argv, _ = self.run_indexer("graphify", pin=bare) - self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + self.assertEqual(argv[3:], ["extract", ".", "--code-only", "--no-cluster"]) def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: # The pin normalizes its own options, so this reaches past the @@ -1201,14 +1331,16 @@ def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: stripped = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) object.__setattr__(stripped, "options", ()) argv, _ = self.run_indexer("graphify", pin=stripped) - self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + self.assertEqual(argv[3:], ["extract", ".", "--code-only", "--no-cluster"]) def test_the_collected_artifact_holds_the_state_the_provider_wrote(self) -> None: _, result = self.run_indexer("graphify") self.assertEqual(result.completeness, lifecycle.COMPLETE) - self.assertEqual(result.indexed_files, 3) + # One of the two census files is a code input to this pin; the Markdown + # file is not, so it is skipped rather than counted against the run. + self.assertEqual(result.indexed_files, 1) names = self.archived_names(self.artifacts[-1].read_bytes()) - self.assertIn("graph.bin", names) + self.assertIn("graph.json", names) def archived_names(self, artifact: bytes) -> list[str]: with tarfile.open(fileobj=io.BytesIO(artifact), mode="r") as archive: @@ -1253,31 +1385,158 @@ def test_a_provider_that_wrote_no_state_publishes_nothing(self) -> None: with self.assertRaises(ContextError): self.run_indexer("graphify", state_directory="") + def test_the_output_is_collected_from_the_root_the_pin_actually_writes(self) -> None: + """``graphify-out/``, not a name this adapter would have preferred. + + The pin builds every output path from ``GRAPHIFY_OUT``, whose default is + the literal ``graphify-out``, so a collector that only accepted + ``.graphify``/``.graph`` found nothing after a real run and refused + every generation the adopted path can produce. + """ + _, result = self.run_indexer("graphify") + self.assertEqual(result.completeness, lifecycle.COMPLETE) + with self.assertRaises(ContextError): + self.run_indexer("graphify", state_directory=".graphify") + + def test_a_second_output_root_beside_the_real_one_publishes_nothing(self) -> None: + """Two roots means the provenance of a generation would be a guess.""" + request = self.request() + + def fake_popen(argv, **kwargs): + for name in (lifecycle._PROVIDER_OUTPUT_DIRECTORY, ".graphify"): + written = request.source_root / name + written.mkdir(exist_ok=True) + (written / "graph.json").write_text("{}", encoding="utf-8") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + + def test_a_symlinked_output_root_publishes_nothing(self) -> None: + """A link names a directory outside the copy; the boundary is the copy.""" + request = self.request() + elsewhere = Path(tempfile.mkdtemp(dir=self.root)) + (elsewhere / "graph.json").write_text("{}", encoding="utf-8") + + def fake_popen(argv, **kwargs): + (request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY).symlink_to(elsewhere) + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + + def test_an_output_root_without_a_graph_document_publishes_nothing(self) -> None: + request = self.request() + + def fake_popen(argv, **kwargs): + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY + written.mkdir(exist_ok=True) + (written / "manifest.json").write_text("{}", encoding="utf-8") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + def test_a_successful_run_with_requeued_entries_is_partial(self) -> None: - # The defect the clean-room run recorded: a repeat that exits zero in - # 1.63 s having requeued 54 entries has not built a complete graph. - _, result = self.run_indexer("graphify", report={"complete": True, "files": 429, "requeued": 54}) + """The defect the clean-room run recorded, in the shape the pin writes it. + + A repeat that exits zero in 1.63 s having requeued 54 entries has not + built a complete graph. The pin records a requeue by blanking the row's + hashes -- ``clear_ast`` does exactly that for an extractor error or an + anomalous zero-node extract -- so a blank ``ast_hash`` is the evidence, + not a ``requeued`` counter the pinned manifest has never carried. + """ + requeued = {CODE_INPUT: {"mtime": 1.0, "seen": 2.0, "ast_hash": "", "semantic_hash": ""}} + _, result = self.run_indexer("graphify", report=requeued) self.assertEqual(result.completeness, lifecycle.PARTIAL) - self.assertIn("54 requeued", " ".join(result.notes)) + self.assertIn("1 code files unprocessed or requeued", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) - def test_a_provider_that_denies_completion_is_partial(self) -> None: - _, result = self.run_indexer("graphify", report={"complete": False, "files": 10}) + def test_a_code_input_the_manifest_never_names_is_partial(self) -> None: + """Silence about a file is not a claim that it was indexed.""" + _, result = self.run_indexer("graphify", report={}) self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) - def test_a_run_that_left_no_report_is_partial_rather_than_complete(self) -> None: - # Exit status zero is not completion evidence. Absent evidence resolves - # to the state ``graph_status`` refuses, not the one it accepts. - _, result = self.run_indexer("graphify", report=None) + def test_a_non_code_input_is_skipped_rather_than_counted_against_the_run(self) -> None: + """The denominator is the pin's own code set, not every tracked file. + + ``docs/guide.md`` is deterministically not code to this pin, so a run + that never touches it is still complete. Holding a correct run to every + tracked documentation file would make it permanently partial. + """ + _, result = self.run_indexer("graphify") + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertNotIn(DOC_INPUT, json.dumps(FINISHED_MANIFEST)) + + def test_a_malformed_manifest_record_is_partial_rather_than_complete(self) -> None: + """A row this adapter cannot read is indistinguishable from a failure.""" + malformed = ( + {CODE_INPUT: "not a row"}, + {CODE_INPUT: {"ast_hash": "a" * 32}}, + {CODE_INPUT: manifest_row("A" * 32)}, + {CODE_INPUT: manifest_row("short")}, + {CODE_INPUT: dict(manifest_row(), ast_hash=None)}, + ) + for report in malformed: + with self.subTest(report=report): + _, result = self.run_indexer("graphify", report=report) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertTrue(result.notes) + + def test_a_manifest_listing_every_code_input_with_a_hash_is_complete(self) -> None: + """Hashes prove processed bytes, which is what completeness claims. + + Nothing else upgrades a run: the exit status is zero in every case here, + and the graph document is ``{}`` -- an empty graph with every input + stamped is a complete read, and a full graph with one input unstamped is + not. The hash has to be *this* input's, which is the next test. + """ + _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row()}) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + + def test_a_hash_that_is_not_this_input_s_bytes_is_partial(self) -> None: + """A hash-shaped string is not a hash of what the provider was given. + + ``b`` repeated is well formed by every rule the old check applied, and it + is not the digest of ``src/app.py``. A row like it is what a manifest + carried over from another tree or resumed from a cache looks like, so it + must not count as work done on these bytes. + """ + _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row("b" * 32)}) self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("not the bytes it was given", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) + + def test_the_expected_digest_is_taken_before_the_provider_runs(self) -> None: + """The comparison is against what this build handed over, not leavings. - def test_an_unparseable_report_is_partial_rather_than_complete(self) -> None: + The stand-in rewrites ``src/app.py`` while it "runs" and stamps the + manifest with the digest of what it wrote. Read after the fact, that + agrees with itself; read before the launch, as it is, it does not. + """ request = self.request() + replacement = b"def main():\n return 1\n" + stamped = hashlib.md5(replacement, usedforsecurity=False).hexdigest() def fake_popen(argv, **kwargs): - written = request.source_root / ".graph" + request.source_root.joinpath(*CODE_INPUT.split("/")).write_bytes(replacement) + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") - (written / "manifest.json").write_bytes(b"{not json") + (written / "graph.json").write_text("{}", encoding="utf-8") + (written / "manifest.json").write_text( + json.dumps({CODE_INPUT: manifest_row(stamped)}), encoding="utf-8" + ) return FakeChild() with stand_in_containment(("/sandbox",)): @@ -1285,41 +1544,401 @@ def fake_popen(argv, **kwargs): with mock.patch.object(subprocess, "Popen", fake_popen): result = indexer(request) self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("not the bytes it was given", " ".join(result.notes)) - def test_a_report_without_affirmative_completion_evidence_is_partial(self) -> None: - """A document that parses is not a document that claims completion. + def test_a_code_input_the_copy_does_not_hold_cannot_be_counted(self) -> None: + """An unreadable input is a row this build cannot check, so it is partial. - ``{}`` and a report in some schema this adapter does not understand - both say nothing about whether the extraction finished, and nothing is - not a claim. Treating them as complete would hand ``graph_status`` a - usable generation built from an unknown run. + Not an error: the census is the denominator and the copy is the + evidence, and a build that lost one of the two states that rather than + believing the provider's row on its own word. """ - silent = ( - {}, - {"schema": "unexpected"}, - {"code_files": 3}, - {"complete": True}, - {"status": "running", "code_files": 3}, - {"status": "partial", "complete": True, "code_files": 3}, + result = lifecycle._read_completeness( + {CODE_INPUT: manifest_row()}, census_of(CODE_INPUT), {} ) - for report in silent: - with self.subTest(report=report): - _, result = self.run_indexer("graphify", report=report) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not re-read 1 code files", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) + + def test_a_named_package_manifest_is_in_the_denominator(self) -> None: + """``classify_file`` routes these by filename before any suffix class. + + ``.toml`` is not a code extension, so an extension-only denominator + drops ``pyproject.toml`` entirely: the provider could fail on it while + ``src/app.py`` alone let the run claim it was complete. Every name the + pin's ``PACKAGE_MANIFEST_NAMES`` carries is checked, including the + mixed-case spellings the repositories that use them actually commit. + """ + for name in ( + "pyproject.toml", "Cargo.toml", "go.mod", "pom.xml", "apm.yml", "apm.yaml", + ): + with self.subTest(name=name): + inputs = lifecycle._provider_inputs(census_of(CODE_INPUT, name)) + self.assertIn(name, inputs.dispatched) + self.assertEqual(inputs.unsupported, frozenset()) + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census_of(CODE_INPUT, name), + {CODE_INPUT: CODE_INPUT_DIGEST}, + inputs, + ) self.assertEqual(result.completeness, lifecycle.PARTIAL) - self.assertTrue(result.notes) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) + + def test_a_shebang_script_without_a_suffix_is_in_the_denominator(self) -> None: + """The pin routes extensionless files by their first line, so this does. - def test_a_report_that_claims_completion_and_counts_its_work_is_complete(self) -> None: - claimed = ( - {"complete": True, "code_files": 3}, - {"status": "success", "indexed_files": 3}, - {"completed": True, "entries": 0, "requeued": 0}, + Both halves matter: a supported interpreter is a dispatched input whose + absence from the manifest keeps the run partial, and one the pin has no + extractor for is dispatched *and* reported as unsupported rather than + counted as work. + """ + root = self.root / "copy" + (root / "bin").mkdir(parents=True) + (root / "bin" / "devctl").write_bytes(b"#!/usr/bin/env python3\nprint(1)\n") + (root / "bin" / "report").write_bytes(b"#!/usr/bin/perl\nprint 1;\n") + (root / "bin" / "notes").write_bytes(b"plain text, no shebang\n") + (root / "bin" / "packed").write_bytes(b"#!/usr/bin/env -S python3 -u\n") + inputs = lifecycle._provider_inputs( + census_of("bin/devctl", "bin/report", "bin/notes", "bin/packed"), root ) - for report in claimed: - with self.subTest(report=report): - _, result = self.run_indexer("graphify", report=report) + self.assertEqual( + set(inputs.dispatched), {"bin/devctl", "bin/report"} + ) + # perl is code to ``detect`` and has no entry in ``_SHEBANG_DISPATCH``. + self.assertEqual(inputs.unsupported, frozenset({"bin/report"})) + # No shebang is a decision, not a refusal: the pin would not call it code. + self.assertNotIn("bin/notes", inputs.unclassified) + # ``env -S`` is the option-carrying form this adapter declines to guess at. + self.assertEqual(inputs.unclassified, ("bin/packed",)) + + def test_an_input_that_cannot_be_classified_keeps_the_run_partial(self) -> None: + """Unknown is not the same as not-code, and must not read as coverage.""" + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census_of(CODE_INPUT), + {CODE_INPUT: CODE_INPUT_DIGEST}, + lifecycle._ProviderInputs( + dispatched=(CODE_INPUT,), unclassified=("bin/packed",) + ), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not classify 1 tracked inputs", " ".join(result.notes)) + + def test_unsupported_dispatch_is_counted_apart_from_indexed_files(self) -> None: + """A stamped row proves read bytes, not a supported extraction. + + ``_get_extractor`` returns ``None`` for a code-classified extension with + no wired extractor, which short-circuits to an empty result carrying + neither an ``error`` nor a ``skipped`` marker -- so the CLI's + failed-source rule leaves the row stamped. Counting that as an indexed + file would report work the provider demonstrably did not do. + """ + census = census_of(CODE_INPUT, "analysis/model.r") + digest = hashlib.md5(b"x", usedforsecurity=False).hexdigest() + result = lifecycle._read_completeness( + {CODE_INPUT: manifest_row(), "analysis/model.r": manifest_row(digest)}, + census, + {CODE_INPUT: CODE_INPUT_DIGEST, "analysis/model.r": digest}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + self.assertEqual(result.unsupported_inputs, 1) + + def test_a_matlab_dot_m_is_dispatched_but_is_not_an_indexed_file(self) -> None: + """``.m`` is the one dispatch the pin decides from the bytes (#1702). + + The suffix table maps ``.m`` to the Objective-C extractor, but + ``_get_extractor`` returns ``None`` for a ``.m`` carrying no + Objective-C directive rather than force-parsing MATLAB through the ObjC + grammar. The row is stamped all the same -- that short circuit carries + neither marker the failed-source rule looks for -- so the static table + difference cannot see it and a stamped row alone would report work the + provider demonstrably did not do. An Objective-C file beside it is a + real extraction and still has to count. + """ + root = self.root / "objc" + (root / "src").mkdir(parents=True) + sources = { + "src/solver.m": b"function y = f(x)\n y = x + 1;\nend\n", + "src/Thing.m": b'#import "Thing.h"\n@implementation Thing\n@end\n', + } + for path, body in sources.items(): + (root / path).write_bytes(body) + census = census_of(*sources) + inputs = lifecycle._provider_inputs(census, root) + self.assertEqual(set(inputs.dispatched), set(sources)) + self.assertEqual(inputs.unsupported, frozenset({"src/solver.m"})) + self.assertEqual(inputs.unclassified, ()) + digests = { + path: hashlib.md5(body, usedforsecurity=False).hexdigest() + for path, body in sources.items() + } + result = lifecycle._read_completeness( + {path: manifest_row(digest) for path, digest in digests.items()}, + census, + digests, + inputs, + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + self.assertEqual(result.unsupported_inputs, 1) + + def test_a_dot_m_the_copy_cannot_be_read_for_stays_unclassified(self) -> None: + """Which way the pin dispatched it is unknown, so the run stays partial. + + The pin's own sniff answers "not Objective-C" when its read fails, but + that is a statement about its read. This adapter failing to read the + same bytes proves nothing, and guessing either way would either excuse a + real extraction or invent an unsupported one. + """ + census = census_of("src/solver.m") + inputs = lifecycle._provider_inputs(census, self.root / "absent") + self.assertEqual(inputs.dispatched, ("src/solver.m",)) + self.assertEqual(inputs.unsupported, frozenset()) + self.assertEqual(inputs.unclassified, ("src/solver.m",)) + digest = hashlib.md5(b"function y = f(x)\n", usedforsecurity=False).hexdigest() + result = lifecycle._read_completeness( + {"src/solver.m": manifest_row(digest)}, + census, + {"src/solver.m": digest}, + inputs, + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not classify 1 tracked inputs", " ".join(result.notes)) + + def test_an_unsupported_extension_still_has_to_be_accounted_for(self) -> None: + """Reported, not excused: an absent row for one is still partial.""" + census = census_of(CODE_INPUT, "web/view.ejs") + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census, + {CODE_INPUT: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) + self.assertEqual(result.unsupported_inputs, 0) + + def test_two_tracked_names_that_differ_only_by_normalization_stay_partial(self) -> None: + """One input's success must never stand in as proof for another's. + + ``src/café.py`` spelled with U+00E9 and the same name spelled ``e`` plus + U+0301 are two Git entries, two blobs and two files, and a Linux + checkout carries both at once. Give them identical bytes -- so their + expected digests are identical too and only the name can tell the rows + apart -- then fail one extraction and stamp the other. Folded onto a + shared key, the stamped row overwrites the blank one and both inputs + read as processed; held apart, neither is counted and the run is what it + actually was. Both census orders and both manifest orders, because a + fix that merely preferred the first or the last colliding row would pass + one of them. + """ + for census_order in ((NFC_INPUT, NFD_INPUT), (NFD_INPUT, NFC_INPUT)): + for rows_order in (census_order, census_order[::-1]): + with self.subTest(census=census_order, manifest=rows_order): + census = census_of(*census_order) + result = lifecycle._read_completeness( + { + rows_order[0]: manifest_row(""), + rows_order[1]: manifest_row(CODE_INPUT_DIGEST), + }, + census, + dict.fromkeys(census_order, CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("Unicode normalization", " ".join(result.notes)) + + def test_colliding_tracked_names_with_different_bytes_stay_partial(self) -> None: + """The digest map collapses the same way the rows do, so it is checked too. + + Different content gives the two inputs different expected digests, and a + map keyed on their shared normal form holds one of them for both. The + row that survives then agrees with whichever digest survived, which is a + comparison between two statements about one file and no statement at all + about the other. + """ + other = hashlib.md5(b"def other():\n return 1\n", usedforsecurity=False).hexdigest() + census = census_of(NFC_INPUT, NFD_INPUT) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), NFD_INPUT: manifest_row(other)}, + census, + {NFC_INPUT: CODE_INPUT_DIGEST, NFD_INPUT: other}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("Unicode normalization", " ".join(result.notes)) + + def test_a_collapsed_manifest_cannot_hide_a_collision_in_the_census(self) -> None: + """The census is the immutable evidence that there were two inputs. + + A provider that wrote one row for the pair leaves nothing in its own + output to say a second file existed. The denominator is not its output: + it is the tracked census taken before the launch, so the missing input + is visible whether or not the manifest admits to it. + """ + census = census_of(NFC_INPUT, NFD_INPUT) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, + dict.fromkeys((NFC_INPUT, NFD_INPUT), CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + + def test_one_tracked_name_still_matches_a_normalized_manifest_key(self) -> None: + """The macOS case normalization was for, which has to keep working. + + A single tracked name whose copy hands the provider a canonically + equivalent spelling is one input and one row. There is nothing to + confuse it with, so it matches and the run is complete -- in both + directions, because which spelling Git holds and which the filesystem + returns are independent. + """ + for tracked, written in ((NFD_INPUT, NFC_INPUT), (NFC_INPUT, NFD_INPUT)): + with self.subTest(tracked=tracked): + census = census_of(tracked) + result = lifecycle._read_completeness( + {written: manifest_row(CODE_INPUT_DIGEST)}, + census, + {tracked: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + + def test_a_manifest_key_spelled_exactly_is_that_input_s_own_record(self) -> None: + """Exact identity is never reached past for a folded near-match. + + Both spellings are present as keys and only one of them is this input's + name. The blank row beside it is about some other file, and a build that + resolved by normal form could read either one as the answer. + """ + census = census_of(NFD_INPUT) + digests = {NFD_INPUT: CODE_INPUT_DIGEST} + inputs = lifecycle._provider_inputs(census) + stamped = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(""), NFD_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, digests, inputs, + ) + self.assertEqual(stamped.completeness, lifecycle.COMPLETE) + self.assertEqual(stamped.indexed_files, 1) + blank = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), NFD_INPUT: manifest_row("")}, + census, digests, inputs, + ) + self.assertEqual(blank.completeness, lifecycle.PARTIAL) + self.assertEqual(blank.indexed_files, 0) + self.assertIn("unprocessed or requeued", " ".join(blank.notes)) + + def test_two_manifest_keys_folding_onto_one_input_are_not_resolved(self) -> None: + """A collision can be the provider's alone, and it is refused the same way. - def test_an_oversized_report_is_refused_without_being_read_whole(self) -> None: + ``U+212B`` (ANGSTROM SIGN) and ``A`` plus ``U+030A`` both fold onto the + ``U+00C5`` the census holds, and neither is spelled the way the tracked + path is. Two candidate records and no exact one is two statements this + build cannot attribute to its single input. + """ + # Escaped, because all three names render identically and a reader + # cannot otherwise tell which line is which. + tracked = "src/Ångstrom.py" + legacy = "src/Ångstrom.py" + decomposed = "src/Ångstrom.py" + # The precondition this test is about, asserted rather than assumed. + self.assertEqual( + {unicodedata.normalize("NFC", key) for key in (legacy, decomposed)}, + {unicodedata.normalize("NFC", tracked)}, + ) + self.assertNotIn(tracked, (legacy, decomposed)) + census = census_of(tracked) + result = lifecycle._read_completeness( + {legacy: manifest_row(""), decomposed: manifest_row(CODE_INPUT_DIGEST)}, + census, + {tracked: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("more than one candidate record", " ".join(result.notes)) + + def test_distinct_unicode_names_that_do_not_fold_together_are_unaffected(self) -> None: + """The refusal is about canonical equivalence, not about non-ASCII names.""" + naive = "src/naïve.py" + census = census_of(NFC_INPUT, naive) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), naive: manifest_row(CODE_INPUT_DIGEST)}, + census, + dict.fromkeys((NFC_INPUT, naive), CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 2) + + def test_input_digests_are_keyed_by_the_exact_tracked_path(self) -> None: + """What the copy was read for is recorded under the name Git holds. + + One file, one spelling, so this runs on any filesystem: the point is the + key, not a dual-name checkout. A map keyed on the normal form would + answer to a different tracked path than the one it was read for, and the + pre-launch digest is the only thing a manifest row is checked against. + The normalized manifest key beside it still resolves, which is the macOS + compatibility this keeps. + """ + root = self.root / "digests" + (root / "src").mkdir(parents=True) + (root / NFD_INPUT).write_bytes(FIXTURE_INPUTS[CODE_INPUT]) + census = census_of(NFD_INPUT) + digests = lifecycle._materialized_digests(root, census) + self.assertEqual(digests, {NFD_INPUT: CODE_INPUT_DIGEST}) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, + digests, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + + def test_a_request_without_a_census_cannot_be_complete(self) -> None: + """There is no denominator, so there is no coverage claim to make.""" + self.assertEqual( + lifecycle._read_completeness(FINISHED_MANIFEST, None).completeness, lifecycle.PARTIAL + ) + + def test_a_build_that_recorded_no_input_digests_cannot_be_complete(self) -> None: + """Without the immutable bytes there is nothing to check a row against.""" + result = lifecycle._read_completeness(FINISHED_MANIFEST, census_of(CODE_INPUT), None) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("no input digests", " ".join(result.notes)) + + def test_a_run_that_left_no_manifest_is_partial_rather_than_complete(self) -> None: + # Exit status zero is not completion evidence. Absent evidence resolves + # to the state ``graph_status`` refuses, not the one it accepts. + _, result = self.run_indexer("graphify", report=None) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_an_unparseable_manifest_is_partial_rather_than_complete(self) -> None: + request = self.request() + + def fake_popen(argv, **kwargs): + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY + written.mkdir(exist_ok=True) + (written / "graph.json").write_text("{}", encoding="utf-8") + (written / "manifest.json").write_bytes(b"{not json") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + result = indexer(request) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_an_oversized_manifest_is_refused_without_being_read_whole(self) -> None: """Provider output is unbounded input; the read is bounded at the stream. Slicing after ``read_bytes()`` would have allocated the whole document @@ -1327,12 +1946,12 @@ def test_an_oversized_report_is_refused_without_being_read_whole(self) -> None: nothing in the collection path may read a provider file whole. """ request = self.request() - oversized = b'{"complete": true, "code_files": 3, "pad": "' + b"x" * lifecycle.MAX_MANIFEST_BYTES + b'"}' + oversized = b'{"' + CODE_INPUT.encode() + b'": "' + b"x" * lifecycle.MAX_MANIFEST_BYTES + b'"}' def fake_popen(argv, **kwargs): - written = request.source_root / ".graphify" + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "graph.json").write_text("{}", encoding="utf-8") (written / "manifest.json").write_bytes(oversized) return FakeChild() @@ -1849,6 +2468,722 @@ def test_a_readable_set_reaching_the_home_directory_is_refused(self) -> None: ) +def macho(path: Path, *dependencies: str, filetype: int = 0x6) -> Path: + """A real 64-bit Mach-O header whose load commands name ``dependencies``. + + A header, not a whole image: the derivation reads ``LC_LOAD_DYLIB`` out of + the load-command block and nothing else, so a file with a real magic, a real + command count and real commands exercises exactly the parse under test + without needing a compiler on the machine running these tests. + + ``filetype`` defaults to ``MH_DYLIB``, which is what every image these tests + write stands in for. It is a parameter because what a candidate *declares + itself to be* is now a check, and the refusals need images that declare + something else. + """ + commands = b"" + for name in dependencies: + raw = name.encode("utf-8") + b"\0" + raw += b"\0" * ((-len(raw)) % 8) + # cmd=LC_LOAD_DYLIB, cmdsize, name offset, timestamp, versions. + commands += struct.pack("<6I", 0x0C, 24 + len(raw), 24, 0, 0, 0) + raw + # magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags, reserved + header = struct.pack( + "<8I", 0xFEEDFACF, 0x0100_000C, 0, filetype, len(dependencies), len(commands), 0, 0 + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + commands) + return path + + +def fat_macho(path: Path, *slices: bytes) -> Path: + """A universal archive carrying ``slices`` as real Mach-O images.""" + header = struct.pack(">2I", 0xCAFEBABE, len(slices)) + offset = len(header) + 20 * len(slices) + arches = b"" + body = b"" + for payload in slices: + arches += struct.pack(">5I", 0x0100_000C, 0, offset + len(body), len(payload), 0) + body += payload + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + arches + body) + return path + + +class LinkedRuntimeLibraryTests(ProviderExposureFixture): + """Shared libraries the exposed runtime links to from outside the exposure. + + A pinned provider's environment and the base interpreter it was created from + were exposed as prefixes, and that was taken to be the whole runtime. On a + host whose interpreter came from a package manager it is not: CPython's + ``_ssl`` extension is linked against an OpenSSL under the *manager's* prefix, + and Graphify imports ``ssl`` during start-up even for a code-only + extraction, so under this boundary the provider aborted before it scanned + anything. + + These hold the repair to the shape it has to have: the dependency is derived + from the images themselves rather than named here, what is exposed is a + library file rather than the manager's prefix, every added path goes through + the same refusals as any other exposure, and Linux is untouched. + """ + + def derive(self, *prefixes: Path) -> tuple[str, ...]: + with mock.patch.object(sys, "platform", "darwin"): + return lifecycle._linked_runtime_libraries( + list(prefixes), covered=list(prefixes), repository=self.repository + ) + + def library(self, path: Path, *dependencies: str) -> Path: + return macho(path, *dependencies) + + def test_a_library_outside_the_exposure_is_added_as_a_file(self) -> None: + brew = self.root / "brew" / "Cellar" / "openssl@3" / "3.6.3" / "lib" + libssl = self.library(brew / "libssl.3.dylib") + prefix = self.root / "python" + self.library(prefix / "lib-dynload" / "_ssl.so", str(libssl)) + derived = self.derive(prefix) + self.assertIn(str(libssl), derived) + # The file, never the directory holding it: exposing that directory is + # exposing a package manager's prefix, and its ``etc`` and ``var`` with + # it. + self.assertNotIn(str(brew), derived) + self.assertNotIn(str(brew.parent), derived) + self.assertNotIn(str(self.root / "brew"), derived) + + def test_a_transitive_dependency_is_reached(self) -> None: + """``libssl`` needs ``libcrypto``, and neither is written down here.""" + brew = self.root / "brew" / "lib" + libcrypto = self.library(brew / "libcrypto.3.dylib") + libssl = self.library(brew / "libssl.3.dylib", str(libcrypto)) + prefix = self.root / "python" + self.library(prefix / "lib-dynload" / "_ssl.so", str(libssl)) + derived = self.derive(prefix) + self.assertIn(str(libssl), derived) + self.assertIn(str(libcrypto), derived) + + def test_a_link_is_resolved_and_both_spellings_are_kept(self) -> None: + """A manager's stable ``opt`` name is a link into its versioned cellar. + + dyld opens the path the image wrote down; a bind-mount boundary names a + literal destination in an otherwise empty root. Neither spelling is the + other, so both are exposed -- and the file that is *validated* is the + resolved one, because a link's own spelling is not what gets read. + """ + cellar = self.library(self.root / "brew" / "Cellar" / "o" / "3" / "lib" / "libssl.dylib") + stable = self.root / "brew" / "opt" / "openssl@3" + stable.parent.mkdir(parents=True, exist_ok=True) + stable.symlink_to(cellar.parent.parent) + referenced = stable / "lib" / "libssl.dylib" + prefix = self.root / "python" + self.library(prefix / "_ssl.so", str(referenced)) + derived = self.derive(prefix) + self.assertIn(str(cellar), derived) + self.assertIn(str(referenced), derived) + + def test_a_dependency_already_inside_the_exposure_is_not_added(self) -> None: + prefix = self.root / "python" + inside = self.library(prefix / "lib" / "libpython.dylib") + self.library(prefix / "lib-dynload" / "_x.so", str(inside)) + self.assertEqual(self.derive(prefix), ()) + + def test_a_dependency_the_read_only_runtime_already_covers_is_not_added(self) -> None: + """``/usr/lib/libSystem.B.dylib`` is in every child's view already.""" + prefix = self.root / "python" + self.library(prefix / "_x.so", "/usr/lib/libSystem.B.dylib") + self.assertEqual(self.derive(prefix), ()) + + def test_loader_relative_names_are_not_treated_as_paths(self) -> None: + """``@rpath`` and friends are resolved by dyld against the image itself.""" + prefix = self.root / "python" + self.library( + prefix / "_x.so", + "@rpath/libfoo.dylib", + "@loader_path/../libbar.dylib", + "@executable_path/libbaz.dylib", + ) + self.assertEqual(self.derive(prefix), ()) + + def test_an_absent_dependency_is_skipped_rather_than_refused(self) -> None: + """A weak link to something this host never installed. + + Skipped, not refused: if it turns out to have been required, the + provider fails at launch as dyld naming the library it could not find, + which is a better answer than this module refusing a build over a link + that is never opened. + """ + prefix = self.root / "python" + self.library(prefix / "_x.so", str(self.root / "brew" / "lib" / "libgone.dylib")) + self.assertEqual(self.derive(prefix), ()) + + def test_a_dependency_inside_the_checkout_is_refused(self) -> None: + """The live working tree is what the materialized copy exists to replace.""" + inside = self.library(self.repository / "vendor" / "libevil.dylib") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(inside)) + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_a_dependency_that_is_the_home_directory_is_refused(self) -> None: + home = self.root / "home" + home.mkdir() + prefix = self.root / "python" + self.library(prefix / "_x.so", str(home)) + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_a_dependency_that_is_not_a_regular_file_is_refused(self) -> None: + directory = self.root / "brew" / "lib" + directory.mkdir(parents=True) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(directory)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a regular file", str(raised.exception)) + + def test_an_implausibly_large_dependency_is_refused(self) -> None: + big = self.root / "brew" / "lib" / "libhuge.dylib" + big.parent.mkdir(parents=True) + big.write_bytes(b"") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(big)) + with mock.patch.object(lifecycle, "_MAX_LINKED_LIBRARY_BYTES", -1): + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("implausibly large", str(raised.exception)) + + def test_a_dependency_another_account_may_rewrite_is_refused(self) -> None: + """What the provider maps executable inside the boundary is code. + + A library some other account may write is a library somebody else + chooses the contents of, and the sandbox would then be confining the + provider to a runtime this host did not install. + """ + loose = self.library(self.root / "brew" / "lib" / "libloose.dylib") + loose.chmod(0o666) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(loose)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("writable by", str(raised.exception)) + + def test_a_dependency_that_is_not_a_binary_is_refused(self) -> None: + """The name comes out of somebody else's image, so the file must answer. + + Ownership and size say who wrote a file and how big it is, not what it + is. Without reading the container, a provider that writes its own linker + input picks which of the operator's files this boundary exposes -- a + shell profile, a keychain database, a notes file -- and each one passes + every metadata check an operator-owned file passes. + """ + secret = self.root / "documents" / "notes.txt" + secret.parent.mkdir(parents=True) + secret.write_text("an operator's file, owned by the operator\n") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(secret)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + + def test_a_truncated_dependency_is_refused(self) -> None: + """A magic is not a header. Nothing is admitted on four bytes.""" + stub = self.root / "brew" / "lib" / "libcut.dylib" + stub.parent.mkdir(parents=True) + stub.write_bytes(b"\xcf\xfa\xed\xfe" + b"\0" * 8) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(stub)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + + def test_a_dependency_that_is_an_executable_is_refused(self) -> None: + """A Mach-O, and still not something a load command may name.""" + binary = macho(self.root / "brew" / "bin" / "tool", filetype=0x2) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(binary)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_dependency_that_is_a_bundle_is_refused(self) -> None: + """``MH_BUNDLE`` is reached through ``dlopen``, not through dyld's loader.""" + bundle = macho(self.root / "brew" / "lib" / "plugin.bundle", filetype=0x8) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(bundle)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_dylib_stub_is_admitted(self) -> None: + """``MH_DYLIB_STUB`` is what a stripped SDK ships in a library's place.""" + stub = macho(self.root / "brew" / "lib" / "libstub.dylib", filetype=0x9) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(stub)) + self.assertIn(str(stub), self.derive(prefix)) + + def test_a_universal_dependency_is_admitted_when_every_slice_is_a_library(self) -> None: + library = self.root / "brew" / "lib" / "libfat.dylib" + fat_macho( + library, + macho(self.root / "arm-slice").read_bytes(), + macho(self.root / "intel-slice").read_bytes(), + ) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(library)) + self.assertIn(str(library), self.derive(prefix)) + + def test_a_universal_dependency_with_a_non_library_slice_is_refused(self) -> None: + """One readable library slice is not a licence for whatever the rest are. + + The child is the provider's interpreter, whose architecture is not + necessarily this one, so the slice that gets loaded inside the boundary + is not the slice a check here would have picked. + """ + mixed = self.root / "brew" / "lib" / "libmixed.dylib" + fat_macho( + mixed, + macho(self.root / "good-slice").read_bytes(), + macho(self.root / "bad-slice", filetype=0x2).read_bytes(), + ) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(mixed)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_universal_dependency_with_an_unreadable_slice_is_refused(self) -> None: + """A fat header may declare a slice the file does not carry.""" + broken = self.root / "brew" / "lib" / "libbroken.dylib" + broken.parent.mkdir(parents=True) + header = struct.pack(">2I", 0xCAFEBABE, 1) + # An architecture record whose offset points past the end of the file. + broken.write_bytes(header + struct.pack(">5I", 0x0100_000C, 0, 4096, 32, 0)) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(broken)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + + def test_more_libraries_than_the_bound_are_refused(self) -> None: + brew = self.root / "brew" / "lib" + names = [str(self.library(brew / f"lib{index}.dylib")) for index in range(4)] + prefix = self.root / "python" + self.library(prefix / "_x.so", *names) + with mock.patch.object(lifecycle, "_MAX_LINKED_LIBRARIES", 2): + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_linux_derives_nothing(self) -> None: + """An ELF runtime's libraries are already under the read-only runtime. + + The derivation reads Mach-O images, of which such a host has none, and + the previous behaviour there is the whole behaviour. + """ + brew = self.root / "brew" / "lib" + libssl = self.library(brew / "libssl.so") + prefix = self.root / "python" + self.library(prefix / "_ssl.so", str(libssl)) + with mock.patch.object(sys, "platform", "linux"): + self.assertEqual( + lifecycle._linked_runtime_libraries( + [prefix], covered=[prefix], repository=self.repository + ), + (), + ) + + def test_a_file_that_is_not_an_image_reads_as_no_dependencies(self) -> None: + prefix = self.root / "python" + prefix.mkdir() + (prefix / "notes").write_bytes(b"not a mach-o at all") + (prefix / "truncated.dylib").write_bytes(b"\xcf\xfa\xed\xfe") + self.assertEqual(self.derive(prefix), ()) + + def test_a_universal_archive_is_read_slice_by_slice(self) -> None: + """python.org ships fat binaries; the union of the slices is the answer.""" + brew = self.root / "brew" / "lib" + first = self.library(brew / "libone.dylib") + second = self.library(brew / "libtwo.dylib") + slices = [ + macho(self.root / "slice-one", str(first)).read_bytes(), + macho(self.root / "slice-two", str(second)).read_bytes(), + ] + header = struct.pack(">2I", 0xCAFEBABE, 2) + offset = len(header) + 40 + body = b"" + arches = b"" + for payload in slices: + arches += struct.pack(">5I", 0x0100_000C, 0, offset + len(body), len(payload), 0) + body += payload + prefix = self.root / "python" + prefix.mkdir(exist_ok=True) + (prefix / "fat.dylib").write_bytes(header + arches + body) + derived = self.derive(prefix) + self.assertIn(str(first), derived) + self.assertIn(str(second), derived) + + def test_the_provider_exposure_carries_the_derived_libraries(self) -> None: + """The whole point: what ``subprocess_indexer`` confines the child to.""" + brew = self.root / "brew" / "lib" + libssl = self.library(brew / "libssl.3.dylib") + environment = self.root / "venv" + provider = self.script(environment / "bin" / "graphify", venv=True) + self.library(environment / "lib" / "_ssl.so", str(libssl)) + with mock.patch.object(sys, "platform", "darwin"): + exposed = self.exposure(provider) + self.assertIn(str(libssl), exposed) + self.assertIn(str(environment), exposed) + self.assertNotIn(str(brew), exposed) + + @unittest.skipUnless(sys.platform == "darwin", "Mach-O linkage is a macOS question") + def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: + """Against this host's actual interpreter, not a fixture. + + The fixtures above prove the parse and the refusals; only this proves + the thing the blocker was about -- that the libraries CPython's ``_ssl`` + really links against end up inside the boundary. Every absolute + dependency that extension names must be either under the read-only + runtime every child gets or in the derived set. + + Not every host runtime can be admitted, and this test must not assume + it. An interpreter installed under a shared package-manager prefix whose + ancestry is group-writable is a runtime another account may rewrite, and + the trust rule refuses it on purpose: the derivation raises rather than + exposing code somebody else chooses the contents of. So a refusal is + checked here rather than swallowed -- this host is asked, independently + of the derivation, whether one of ``_ssl``'s own dependencies really is + untrusted, and the refusal is only accepted when it is. A refusal over a + runtime this host does trust would be the regression this test exists to + catch, and any other exception is not caught at all. + """ + import _ssl # noqa: PLC0415 - the point is this host's real extension + + extension = Path(getattr(_ssl, "__file__", "") or "") + if not extension.is_file(): # pragma: no cover - a statically linked build + self.skipTest("this interpreter's _ssl is not a separate extension module") + prefix = Path(sys.base_prefix) + system = [Path(os.path.realpath(path)) for path in lifecycle._SYSTEM_READ_PATHS] + covered = [*system, Path(os.path.realpath(prefix))] + # The extension's own out-of-runtime dependencies, resolved the way the + # derivation resolves them. Computed before the derivation runs so the + # expectation does not come from the code under test. + external = [ + Path(os.path.realpath(name)) + for name in lifecycle._macho_dylib_names(extension) + if name.startswith("/") + ] + external = [ + resolved + for resolved in external + if not any(lifecycle._under(resolved, root) for root in covered) + ] + untrusted = [str(path) for path in external if not lifecycle._trusted_library(path)] + # The same question for the other refusal a real host can legitimately + # hit: a dependency that resolves to something which does not declare + # itself a shared library. Asked here so a refusal over a host whose + # dependencies *are* all libraries still fails the test. + malformed = [ + str(path) + for path in external + if (types := lifecycle._macho_filetypes(path)) is None + or any(kind not in lifecycle._MACHO_DYLIB_FILETYPES for kind in types) + ] + try: + derived = lifecycle._linked_runtime_libraries( + [prefix], covered=[prefix], repository=self.repository + ) + except lifecycle.ContextError as refusal: + # The derivation walks the whole runtime, so the library it refused + # need not be one of ``_ssl``'s: a shared package-manager prefix is + # refused through its own ancestry, before any single dependency. + # Either is a correct refusal; neither being true is not. + self.assertTrue( + untrusted + or malformed + or not lifecycle._trusted_library(Path(os.path.realpath(prefix))), + "the boundary refused this host's runtime, but the runtime and " + f"every library _ssl links to are trusted shared libraries: {refusal}", + ) + return + self.assertEqual( + untrusted, + [], + "the boundary admitted a runtime whose libraries are writable by another account", + ) + self.assertEqual( + malformed, + [], + "the boundary admitted a dependency that is not a shared library", + ) + for resolved in external: + self.assertIn( + str(resolved), derived, f"{resolved} is outside the provider's boundary" + ) + + +def fat_container( + path: Path, + *slices: bytes, + order: str = ">", + magic: int = 0xCAFEBABE, + count: int | None = None, + extents: list[tuple[int, int]] | None = None, +) -> Path: + """A universal archive with every field of its header under the test's hand. + + :func:`fat_macho` writes a well-formed one. This writes whatever a malformed + or hostile one would say: a count that disagrees with the records, an offset + inside the header being read, a slice running off the end of the file, two + slices claiming the same bytes, or the whole header in the other byte order. + ``extents`` replaces the computed ``(offset, size)`` of each record; the + payloads are still laid out end to end after the table, so a record can + describe a region the file does or does not carry. + """ + declared = len(slices) if count is None else count + header = struct.pack(f"{order}2I", magic, declared) + start = len(header) + 20 * len(slices) + body = b"" + placed = [] + for payload in slices: + placed.append((start + len(body), len(payload))) + body += payload + records = extents if extents is not None else placed + arches = b"" + for offset, size in records: + arches += struct.pack(f"{order}5I", 0x0100_000C, 0, offset, size, 0) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + arches + body) + return path + + +class MachoContainerStructureTests(unittest.TestCase): + """What a candidate's own container has to establish before it is exposed. + + Ownership, size and regular-file status say who wrote a file and how big it + is. A magic and a ``filetype`` field say what four bytes and one integer + claim -- and the name being checked came out of an ``LC_LOAD_DYLIB`` in the + provider's own image, so those are the cheapest thing in the file for a + provider that writes its own linker input to reproduce over arbitrary + operator-owned bytes. + + These hold the parse to the structure behind the claim: the container is + decoded in the byte order its magic declares, every declared region is + checked against the length of the file that carries it, slices may not + overlap each other or the table describing them, and each image's + load-command region must be present and walk to exactly the size its header + declared. A container that fails any of it is refused rather than read as + far as it parses. + """ + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.addCleanup(self._directory.cleanup) + self.root = Path(self._directory.name).resolve() + + def thin(self, *dependencies: str, filetype: int = 0x6) -> bytes: + return macho(self.root / "payload", *dependencies, filetype=filetype).read_bytes() + + # -- containers that are read ------------------------------------------ + + def test_a_thin_library_reads_as_its_filetype_and_dependencies(self) -> None: + library = macho(self.root / "libthin.dylib", "/usr/lib/libz.dylib") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6,)) + self.assertEqual(lifecycle._macho_dylib_names(library), ("/usr/lib/libz.dylib",)) + + def test_a_universal_library_reads_as_every_slice(self) -> None: + library = fat_container(self.root / "libfat.dylib", self.thin(), self.thin(filetype=0x9)) + self.assertEqual(lifecycle._macho_filetypes(library), (0x6, 0x9)) + + def test_a_swapped_universal_header_is_decoded_in_its_own_order(self) -> None: + """``FAT_CIGAM`` is the same header written the other way round. + + Read big-endian -- which is what this did -- a count of two reads as + 33_554_432 and every slice offset is a number with no relation to the + file, so a legitimate archive was refused as malformed and the fields + that were supposed to bound the parse were never the fields on disk. + """ + library = fat_container( + self.root / "libswapped.dylib", + self.thin(), + self.thin(filetype=0x9), + order="<", + ) + # The same header value, written the other way: ``FAT_CIGAM`` on disk. + self.assertEqual(library.read_bytes()[:4], b"\xbe\xba\xfe\xca") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6, 0x9)) + + def test_a_swapped_universal_header_carries_its_dependencies(self) -> None: + """The same decode, through the derivation that reads load commands.""" + library = fat_container( + self.root / "libswapped.dylib", + self.thin("/usr/lib/libz.dylib"), + order="<", + ) + self.assertEqual(lifecycle._macho_dylib_names(library), ("/usr/lib/libz.dylib",)) + + def test_an_image_with_no_load_commands_is_read(self) -> None: + """Zero commands and a zero-length region agree with each other.""" + library = macho(self.root / "libbare.dylib") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6,)) + + # -- containers that are refused --------------------------------------- + + def refused(self, path: Path) -> None: + self.assertIsNone(lifecycle._macho_filetypes(path)) + self.assertEqual(lifecycle._macho_dylib_names(path), ()) + + def test_a_file_that_is_not_a_macho_is_refused(self) -> None: + other = self.root / "notes.txt" + other.write_bytes(b"not a mach-o at all") + self.refused(other) + + def test_a_sixty_four_bit_universal_header_is_refused_as_unrecognized(self) -> None: + """``FAT_MAGIC_64`` has wider records; guessing at them is not reading them.""" + container = self.root / "lib64fat.dylib" + container.write_bytes(struct.pack(">2I", 0xCAFEBABF, 1) + b"\0" * 32) + self.refused(container) + + def test_a_slice_running_past_the_end_of_the_file_is_refused(self) -> None: + payload = self.thin() + container = fat_container( + self.root / "libpast.dylib", payload, extents=[(28, len(payload) + 4096)] + ) + self.refused(container) + + def test_a_slice_beginning_past_the_end_of_the_file_is_refused(self) -> None: + container = fat_container( + self.root / "libgone.dylib", self.thin(), extents=[(0x10_0000, 32)] + ) + self.refused(container) + + def test_a_slice_inside_the_architecture_table_is_refused(self) -> None: + """A slice may not start in the header that is describing it.""" + payload = self.thin() + container = fat_container( + self.root / "liboverlap.dylib", payload, extents=[(4, len(payload))] + ) + self.refused(container) + + def test_a_zero_length_slice_is_refused(self) -> None: + container = fat_container(self.root / "libempty.dylib", self.thin(), extents=[(28, 0)]) + self.refused(container) + + def test_overlapping_slices_are_refused(self) -> None: + """Two records claiming the same bytes make "which image is this" ambiguous.""" + payload = self.thin() + container = fat_container( + self.root / "libambiguous.dylib", + payload, + payload, + extents=[(48, len(payload)), (48 + len(payload) // 2, len(payload))], + ) + self.refused(container) + + def test_an_architecture_table_larger_than_the_file_is_refused(self) -> None: + """A count is a promise about bytes the file has to carry.""" + container = fat_container(self.root / "libclaims.dylib", self.thin(), count=8) + self.refused(container) + + def test_more_architectures_than_the_bound_are_refused(self) -> None: + container = self.root / "libmany.dylib" + count = lifecycle._MAX_MACHO_ARCHITECTURES + 1 + container.write_bytes(struct.pack(">2I", 0xCAFEBABE, count) + b"\0" * (20 * count)) + self.refused(container) + + def test_a_zero_architecture_universal_header_is_refused(self) -> None: + container = self.root / "libnone.dylib" + container.write_bytes(struct.pack(">2I", 0xCAFEBABE, 0)) + self.refused(container) + + def test_a_truncated_thin_header_is_refused(self) -> None: + container = self.root / "libcut.dylib" + container.write_bytes(b"\xcf\xfa\xed\xfe" + b"\0" * 8) + self.refused(container) + + def test_a_truncated_load_command_region_is_refused(self) -> None: + """The header declares a region; the file has to carry all of it.""" + whole = self.thin("/usr/lib/libz.dylib") + container = self.root / "libshort.dylib" + container.write_bytes(whole[:-8]) + self.refused(container) + + def test_a_load_command_region_declared_past_the_slice_is_refused(self) -> None: + """A slice may not reach into the next slice's bytes to satisfy its header. + + The bytes that follow are a real, complete image, so a parse bounded by + the file rather than by the slice would read them and admit this. + """ + payload = self.thin() + stretched = bytearray(payload) + struct.pack_into(" None: + """Every command carries at least its own command and size.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + """``sizeofcmds`` is the size of *all* the commands, not of a prefix. + + A chain that leaves bytes over means the region holds something other + than the commands it was declared to hold, and reading the prefix as if + it were the whole truth is how a trailing record goes unexamined. + """ + image = bytearray(self.thin("/usr/lib/libz.dylib", "/usr/lib/libiconv.dylib")) + struct.pack_into(" None: + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + """dyld requires each ``cmdsize`` to be a multiple of the pointer width.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + declared = struct.unpack_from(" None: + """A command of no size is an unbounded walk, not a record.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + image = bytearray(self.thin()) + struct.pack_into(" None: + """One readable slice is not a licence for whatever the rest are.""" + broken = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: manifest = self.build() self.assertEqual(lifecycle.load_manifest(manifest.to_json()), manifest) + def test_the_unsupported_count_survives_the_build_and_the_status_report(self) -> None: + """The number has to outlive the indexer, or it is not a report. + + ``IndexResult.notes`` are read by the build and then dropped, so a + count that lived only there would never reach an operator. This one is + written into the manifest, round-trips through validation, and appears + in the summary ``status`` emits. + """ + manifest = self.build( + indexer=recording_indexer(indexed_files=1, unsupported_inputs=2) + ) + self.assertEqual(manifest.unsupported_inputs, 2) + self.assertEqual(manifest.to_json()["unsupported_inputs"], 2) + self.assertEqual(lifecycle.load_manifest(manifest.to_json()), manifest) + self.assertEqual(manifest.shareable_summary()["unsupported_inputs"], 2) + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertIsNotNone(status.manifest) + self.assertEqual(status.shareable_summary()["build"]["unsupported_inputs"], 2) + + def test_the_unsupported_count_is_not_the_unmaterialized_count(self) -> None: + """Two different questions, so two different numbers. + + ``skipped_paths`` is what this build declined to materialize; the new + count is what the provider could not extract from what it *was* given. + Folding either into the other would report a number an operator cannot + act on. + """ + manifest = self.build( + indexer=recording_indexer(indexed_files=1, unsupported_inputs=1) + ) + self.assertEqual(manifest.skipped_paths, 0) + self.assertEqual(manifest.unsupported_inputs, 1) + + def test_a_manifest_published_before_the_count_existed_still_loads(self) -> None: + """Compatibility, bounded to exactly the one new name.""" + payload = self.build().to_json() + payload.pop("unsupported_inputs") + self.assertEqual(lifecycle.load_manifest(payload).unsupported_inputs, 0) + payload["some_other_field"] = 1 + with self.assertRaises(ContextError): + lifecycle.load_manifest(payload) + def test_shareable_summary_carries_no_content_or_local_path(self) -> None: summary = self.build().shareable_summary() rendered = json.dumps(summary) @@ -3043,10 +4420,51 @@ def indexer_script(self, *, complete: bool = True, **installed) -> Path: body=( "#!/bin/sh\n" '[ "$1" = "extract" ] || exit 64\n' - "mkdir -p .graphify\n" - "printf graph-bytes > .graphify/graph.bin\n" - 'printf \'{"complete": %s, "code_files": 1, "requeued": 0}\' ' - f"'{'true' if complete else 'false'}' > .graphify/manifest.json\n" + # The pinned CLI requires a scan target here and exits 1 without + # one, so the stand-in refuses the same argv the real provider + # refuses rather than accepting a launch that could never work. + '[ -n "$2" ] || exit 65\n' + 'case "$2" in -*) exit 65 ;; esac\n' + '[ -d "$2" ] || exit 65\n' + # The real output root, with the two documents the pinned + # ``--no-cluster`` branch writes there. + "mkdir -p graphify-out\n" + 'printf \'{"nodes": [], "edges": []}\' > graphify-out/graph.json\n' + # A complete run stamps every input it was shown; a partial one + # leaves the manifest empty, which is what an unprocessed code + # file looks like to the collector. + + ( + ( + # The real pin stamps ``_md5_file`` of the bytes it was + # handed, and the collector checks the row against the + # immutable materialized copy. A constant here would be + # a stand-in that never reads its inputs, so it stamps + # the actual digest -- ``md5 -q`` on macOS, ``md5sum`` + # elsewhere -- and a build/refresh round trip exercises + # the same contract a real provider has to meet. + # Three spellings because the host decides which exists: + # openssl and md5 ship with macOS, md5sum with GNU + # coreutils. An empty result is left empty rather than + # faked, so a host with none of them fails the coverage + # check loudly instead of passing on a constant. + "digest_of() {\n" + " h=$(openssl dgst -md5 -r \"$1\" 2>/dev/null | cut -d' ' -f1)\n" + " [ -n \"$h\" ] || h=$(md5 -q \"$1\" 2>/dev/null)\n" + " [ -n \"$h\" ] || h=$(md5sum \"$1\" 2>/dev/null | cut -d' ' -f1)\n" + " printf '%s' \"$h\"\n" + "}\n" + "{ printf '{'; sep=''; " + "find . -path ./graphify-out -prune -o -type f -print | " + "sed 's|^\\./||' | while read -r f; do " + 'h=$(digest_of "$f"); ' + 'printf \'%s"%s":{"mtime":1,"seen":2,' + '"ast_hash":"%s",' + "\"semantic_hash\":\"%s\"}' \"$sep\" \"$f\" \"$h\" \"$h\"; " + "sep=','; done; printf '}'; } > graphify-out/manifest.json\n" + ) + if complete + else "printf '{}' > graphify-out/manifest.json\n" + ) ), **installed, ) diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py new file mode 100644 index 00000000..d1ca98e3 --- /dev/null +++ b/tests/test_context_graph_query.py @@ -0,0 +1,2032 @@ +"""Offline tests for bounded local-graph queries and packets (issue #914). + +Every test builds a real throwaway Git repository, publishes a generation +through the #913 lifecycle with an injected indexer, and asks the resulting +graph a question. No graph package is installed, imported, or required: the +graph document is written by the injected indexer, so what is proved here is +what this repository is responsible for -- how a traversal is bounded, what a +packet binds, which citations survive validation against the immutable tree, +and what happens when the graph cannot answer. + +The graph content is synthetic, as in ``tests/fixtures/local_graph_contract.json``. +A passing suite is not evidence that any provider emits this shape. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import subprocess +import tarfile +import tempfile +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from code_mower import context_contract as contract +from code_mower import context_delivery +from code_mower import context_graph +from code_mower import context_graph_command as command +from code_mower import context_graph_lifecycle as lifecycle +from code_mower import context_graph_query as query +from code_mower.context_contract import ContextError + + +NOW = datetime(2026, 1, 1, 12, tzinfo=timezone.utc) +PIN = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) + +#: Line counts the synthetic graph cites into. Every span below is inside them, +#: except where a test deliberately claims past the end of a file. +SOURCES = { + "example_pkg/config.py": 40, + "example_pkg/loader.py": 50, + "example_pkg/report.py": 20, + "tests/test_config.py": 30, +} + + +def git(repository: Path, *arguments: str) -> None: + subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + env={ + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", + "GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_NAME": "Test", "GIT_COMMITTER_EMAIL": "test@example.invalid", + "PATH": "/usr/bin:/bin:/usr/local/bin", "HOME": str(repository), + }, + ) + + +def make_repository(root: Path) -> Path: + repository = root / "checkout" + repository.mkdir() + git(repository, "init", "-q", "-b", "main") + for relative, lines in SOURCES.items(): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(f"line {n}" for n in range(1, lines + 1)) + "\n", encoding="utf-8") + git(repository, "add", ".") + git(repository, "commit", "-q", "-m", "initial") + return repository + + +def node(identifier: str, label: str, path: str, line=None, **extra) -> dict: + """One node in the pinned exporter's own shape. + + The field names and the annotations are the ones Graphify writes at the + pinned commit: the four required fields of ``graphify/validate.py`` + (``id``, ``label``, ``file_type``, ``source_file``), the extractor's + ``source_location`` of the form ``L``, and the ``community`` / + ``community_name`` / ``norm_label`` annotations ``export.py::to_json`` + adds to every node on the way out. No Code Mower node ``kind`` and no + line span: neither exists in the real export. + """ + return { + "id": identifier, + "label": label, + "file_type": "code", + "source_file": path, + "source_location": "" if line is None else f"L{line}", + "community": 0, + "community_name": "Community 0", + "norm_label": label.lower(), + **extra, + } + + +def edge(source: str, target: str, relation: str, confidence: str = "EXTRACTED", **extra) -> dict: + """One link in the pinned exporter's own shape. + + The five required edge fields, the uppercase ``confidence`` vocabulary of + the pinned validator, and the ``weight`` / ``confidence_score`` the + extractor and exporter attach. ``confidence_score`` uses the exporter's + own ``_CONFIDENCE_SCORE_DEFAULTS``. + """ + scores = {"EXTRACTED": 1.0, "INFERRED": 0.55, "AMBIGUOUS": 0.2} + return { + "source": source, + "target": target, + "relation": relation, + "confidence": confidence, + "source_file": "example_pkg/config.py", + "source_location": "L12", + "weight": 1.0, + "confidence_score": scores[confidence], + **extra, + } + + +def graph_nodes() -> list: + """A symbol, its caller, its caller's caller, a test, and a file node.""" + return [ + node("n-config", "parse_config", "example_pkg/config.py", 12), + node("n-load", "load", "example_pkg/loader.py", 40), + node("n-report", "render", "example_pkg/report.py", 5), + node("n-test", "test_parse_config", "tests/test_config.py", 8), + # The extractor's per-file node: label is the file's base name at L1. + node("n-config-file", "config.py", "example_pkg/config.py", 1), + ] + + +def graph_edges() -> list: + return [ + edge("n-load", "n-config", "calls"), + edge("n-report", "n-load", "calls", "INFERRED"), + edge("n-test", "n-config", "tests"), + edge("n-config-file", "n-config", "contains"), + ] + + +def graph_document(**extra) -> dict: + """A small graph in the format the lifecycle's pinned invocation writes. + + ``context_graph_lifecycle`` requires ``extract --code-only --no-cluster``, + and the pinned CLI's ``--no-cluster`` branch dumps the merged extractor + result straight to ``graph.json`` through ``write_json_atomic``. So the + top-level shape is the raw extraction: ``nodes``, ``edges``, ``hyperedges``, + the token counters and ``extracted_sources``. + + What it deliberately does **not** carry is a ``directed`` marker, or + ``multigraph``, ``graph``, ``links`` or ``built_at_commit``. That path never + constructs a NetworkX graph and never calls ``export.py::to_json``, so none + of those keys exist in a real generation, and a fixture that added one to + satisfy the reader would be testing a file the provider never writes. + """ + return { + "nodes": graph_nodes(), + "edges": graph_edges(), + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + "extracted_sources": sorted(SOURCES), + **extra, + } + + +def wide_graph_document(callers: int = 6, **extra) -> dict: + """``parse_config`` with ``callers`` distinct, separately citable callers. + + Every caller is a real node at its own line of an indexed file, so each + relationship the impact query reports carries two citations the bound commit + confirms. Nothing here is uncitable, duplicated, or dangling: the only + reason a document can go missing from a packet built over this graph is a + budget, which is what the tests using it are about. + """ + lines = SOURCES["example_pkg/loader.py"] + if callers + 1 > lines: # pragma: no cover - guards the fixture, not the code + raise AssertionError("the fixture file has no line left to cite") + return { + "nodes": [ + node("n-config", "parse_config", "example_pkg/config.py", 12), + *( + node(f"n-caller-{index}", f"caller_{index}", "example_pkg/loader.py", index + 1) + for index in range(1, callers + 1) + ), + ], + "edges": [edge(f"n-caller-{index}", "n-config", "calls") for index in range(1, callers + 1)], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + "extracted_sources": sorted(SOURCES), + **extra, + } + + +def node_link_document(**extra) -> dict: + """The same graph as ``export.py::to_json`` writes it, for the clustered path. + + ``networkx.json_graph.node_link_data(G, edges="links")`` plus the + ``hyperedges`` list and the ``built_at_commit`` stamp the exporter appends. + Directed, because this format's endpoint order is only a caller/callee claim + when the graph was stored as a ``DiGraph``. + """ + return { + "directed": True, + "multigraph": False, + "graph": {}, + "nodes": graph_nodes(), + "links": graph_edges(), + "hyperedges": [], + **extra, + } + + +def artifact(document: dict) -> bytes: + """The generation artifact: the provider's state, packed as the lifecycle packs it.""" + body = json.dumps(document).encode() + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w", format=tarfile.PAX_FORMAT) as archive: + info = tarfile.TarInfo(query.GRAPH_MEMBER) + info.size = len(body) + info.mtime = 0 + info.mode = 0o600 + archive.addfile(info, io.BytesIO(body)) + return buffer.getvalue() + + +def indexer(document: dict, *, completeness: str = lifecycle.COMPLETE): + payload = artifact(document) + + def run(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: + request.output_path.write_bytes(payload) + return lifecycle.IndexResult(completeness=completeness, indexed_files=len(SOURCES)) + + return run + + +def envelope(**overrides) -> dict: + value = { + "schema": contract.CONNECTION_SCHEMA, + "capability_version": 1, + "connection": "example-context", + "provider": "synthetic-graph", + "kind": "repository", + "generation": "generation-one", + "state": "verified", + "identity": {"repository_root": "/example/repository"}, + "repositories": ["owner/repo"], + "recipients": ["claude:builder", "codex:reviewer", "devin:builder"], + "expires_at": "2026-01-01T13:00:00Z", + "capabilities": {"search": True, "memory": False, "revision_binding": True}, + } + value.update(overrides) + return value + + +def policy(**overrides) -> dict: + value = { + "schema": contract.POLICY_SCHEMA, + "connection": "example-context", + "policy_version": "v1", + "required": True, + } + value.update(overrides) + return value + + +class GraphWorkspace(unittest.TestCase): + """A published generation over a real commit, with an injected graph.""" + + document: dict = {} + completeness = lifecycle.COMPLETE + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.addCleanup(self._directory.cleanup) + self.root = Path(self._directory.name).resolve() + self.state = self.root / "state" + self.repository = make_repository(self.root) + self.manifest = self.publish(self.document or graph_document()) + + def publish(self, document: dict, *, completeness: str | None = None): + return lifecycle.build_graph( + self.repository, + pin=PIN, + indexer=indexer(document, completeness=completeness or self.completeness), + root=self.state, + now=NOW, + ) + + def authorized(self, **overrides) -> dict: + """An envelope as the connection mints one: bound to what is published now. + + ``authorize_locked`` reads the current generation on every call rather + than remembering one, and a packet may only carry the generation its + evidence came from. A literal fixed at ``setUp`` would authorize one + generation and cite another as soon as a test rebuilds the graph -- + the disagreement the packet builder refuses. + """ + published = lifecycle.graph_status(self.repository, root=self.state).generation + return envelope(**{"generation": published, **overrides}) + + def context(self, **overrides) -> query.GraphContext: + arguments = { + "question": "impact", + "target": "parse_config", + "envelope": self.authorized(), + "policy": policy(), + "context_repository": "owner/repo", + "work_item": "work-item-one", + "root": self.state, + "now": NOW, + } + arguments.update(overrides) + return query.graph_context(self.repository, **arguments) + + def graph(self) -> query.CodeGraph: + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + return query.read_graph(state, lifecycle.graph_status(self.repository, root=self.state)) + + def load(self, packet: dict, *, recipient: str, revision: str | None): + """Load one packet through the shared delivery contract, as a recipient does.""" + encoded = json.dumps(packet).encode() + target = self.root / "delivered-packet.json" + target.write_bytes(encoded) + target.chmod(0o600) + return contract.load_packet( + private_root=self.root, + reference={"path": "delivered-packet.json", "sha256": hashlib.sha256(encoded).hexdigest()}, + policy=policy(), + request=contract.ContextRequest("owner/repo", "work-item-one", recipient, revision), + authorize=lambda: self.authorized(), + now=NOW, + ) + + +class GraphSchemaTests(unittest.TestCase): + """The pinned provider's own output is what gets read, and read bounded. + + The default fixture is the raw extraction the lifecycle's own pinned + ``extract --code-only --no-cluster`` writes. The node-link export the + clustered path writes is covered separately in ``NodeLinkFormatTests``, + because its provenance and its direction guarantee are different and must + not be tested by editing this fixture into that shape. The tests split into + two halves on purpose: what a real generation carries must load, and what + the provider's own validator would reject must refuse. + """ + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_reads_the_pinned_provider_export(self) -> None: + graph = self.load(graph_document()) + self.assertEqual(len(graph.nodes), 5) + # One line per node, because ``source_location`` records one line. + self.assertEqual(graph.nodes["n-config"].citation, "example_pkg/config.py#L12") + self.assertEqual( + {node.id: node.kind for node in graph.nodes.values()}, + { + "n-config": "symbol", "n-load": "symbol", "n-report": "symbol", + # Derived: label equals the file's base name. + "n-config-file": "file", + # Derived: the repository's test layout. + "n-test": "test", + }, + ) + + def test_keeps_the_providers_own_relation_and_lowercases_confidence(self) -> None: + """``contains`` filters as ``defines`` but still reads as ``contains``.""" + graph = self.load(graph_document()) + by_pair = {(edge.source, edge.target): edge for edge in graph.edges} + contains = by_pair[("n-config-file", "n-config")] + self.assertEqual((contains.relation, contains.kind), ("contains", "defines")) + self.assertEqual(by_pair[("n-report", "n-load")].evidence, "inferred") + + def test_reads_the_raw_extraction_without_a_directed_marker(self) -> None: + """The supported document declares no direction, and must not be asked to. + + The pinned ``--no-cluster`` branch writes the merged extractor result + directly: no NetworkX graph is built, ``to_json`` is never called, and + so ``directed``, ``multigraph``, ``graph`` and ``built_at_commit`` are + absent from every real generation. A reader that demanded the marker + would refuse the only path the lifecycle actually runs. + """ + document = graph_document() + for absent in ("directed", "multigraph", "graph", "links", "built_at_commit"): + self.assertNotIn(absent, document) + graph = self.load(document) + self.assertEqual(len(graph.edges), 4) + # The orientation is the extractor's, and it is the one queried. + self.assertEqual( + {(item.source, item.target) for item in graph.edges if item.relation == "calls"}, + {("n-load", "n-config"), ("n-report", "n-load")}, + ) + + def test_reversed_node_iteration_does_not_reverse_a_raw_edge(self) -> None: + """A raw edge's endpoints come off the edge record, not from node order. + + This is the property the ``--no-cluster`` path has and an undirected + NetworkX container does not. ``add_edge`` writes the call site's own + source and target, so permuting the node list -- the only thing an + undirected container's endpoint order would follow -- cannot change + which way a relationship points. + """ + forward = self.load(graph_document()) + reversed_nodes = graph_document() + reversed_nodes["nodes"] = list(reversed(reversed_nodes["nodes"])) + permuted = self.load(reversed_nodes) + self.assertEqual( + [(item.source, item.target, item.relation) for item in forward.edges], + [(item.source, item.target, item.relation) for item in permuted.edges], + ) + # And the claim itself, stated the way a packet states it. + calls = next(item for item in permuted.edges if item.target == "n-config" + and item.relation == "calls") + self.assertEqual((calls.source, calls.target), ("n-load", "n-config")) + + def test_maps_an_unlisted_relation_without_asserting_a_listed_one(self) -> None: + """An LLM-extracted relation is carried, grouped as ``related``, never renamed.""" + document = graph_document() + document["edges"].append(edge("n-config", "n-report", "supersedes")) + graph = self.load(document) + extra = next(edge for edge in graph.edges if edge.relation == "supersedes") + self.assertEqual(extra.kind, query.OTHER_RELATION) + # ``related`` is not in the impact filter, so it cannot stand in for a call. + self.assertNotIn(query.OTHER_RELATION, query._TRAVERSALS["impact"][1]) + + def test_keeps_a_sourceless_stub_traversable_and_uncitable(self) -> None: + """The extractor's cross-file stub: a real node with no location.""" + document = graph_document() + document["nodes"].append({ + "id": "n-stub", "label": "Thing", "file_type": "code", + "source_file": "", "source_location": "", "origin_file": "example_pkg/config.py", + }) + document["edges"].append(edge("n-config", "n-stub", "references")) + graph = self.load(document) + self.assertIsNone(graph.nodes["n-stub"].citation) + self.assertEqual(len(graph.edges), 5) + + def test_drops_non_code_corpora_and_prunes_their_edges(self) -> None: + """Documents and concepts are not repository relationships.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["edges"].append(edge("n-config", "n-doc", "references")) + graph = self.load(document) + self.assertNotIn("n-doc", graph.nodes) + self.assertEqual(len(graph.edges), 4) + + def test_tolerates_provider_annotations_it_does_not_read(self) -> None: + """Extra exporter and LLM metadata must not reject a real generation.""" + document = graph_document() + document["nodes"][0]["metadata"] = {"namespace": "example_pkg", "scope_chain": ["mod"]} + document["nodes"][0]["type"] = "namespace" + document["edges"][0]["context"] = "call site" + self.assertEqual(len(self.load(document).nodes), 5) + + def test_refuses_a_document_with_no_provider_nodes_and_edges(self) -> None: + for document in ( + {"nodes": []}, {"edges": []}, {"links": []}, {"schema": "something.else"}, [], + ): + with self.subTest(document=document): + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_a_graph_built_from_another_commit(self) -> None: + """``built_at_commit`` disagreeing with the generation is a refusal. + + The stamp is ``to_json``'s, so the raw path never writes one and for a + real ``--no-cluster`` generation this check is vacuous -- the binding + rests on the lifecycle's commit binding and the citation census. It is + still honoured wherever it appears, which is what this covers. + """ + with self.assertRaises(ContextError): + self.load(graph_document(built_at_commit="c" * 40)) + # Agreeing is fine, and is the ordinary case. + self.assertEqual(len(self.load(graph_document(built_at_commit="b" * 40)).nodes), 5) + + def test_refuses_records_missing_the_providers_required_fields(self) -> None: + for mutate in ( + lambda doc: doc["nodes"][0].pop("label"), + lambda doc: doc["nodes"][0].pop("source_file"), + lambda doc: doc["nodes"][0].pop("file_type"), + lambda doc: doc["edges"][0].pop("relation"), + lambda doc: doc["edges"][0].pop("confidence"), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_vocabularies_the_providers_validator_rejects(self) -> None: + for mutate in ( + lambda doc: doc["nodes"][0].update(file_type="diagram"), + lambda doc: doc["edges"][0].update(confidence="GUESSED"), + # Lowercase is the packet contract's vocabulary, not the provider's. + lambda doc: doc["edges"][0].update(confidence="extracted"), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_a_vocabulary_field_that_is_not_text_at_all(self) -> None: + """A JSON array or object where a word belongs is a refusal, not a crash. + + Both vocabulary fields are checked by membership in a set or a dict, and + an unhashable value there raises ``TypeError`` -- out of a reader whose + callers only ever catch ``ContextError``, so the graph-context path + would propagate it instead of reporting the graph unreadable. + """ + for field_name, record in (("file_type", "nodes"), ("confidence", "edges")): + for value in ([], {}, ["code"], {"value": "EXTRACTED"}, 3, None, True): + with self.subTest(field=field_name, value=value): + document = graph_document() + document[record][0][field_name] = value + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_an_unreadable_source_location(self) -> None: + for location in ("12", "line 12", "L", "L0", "L-4", "L99999999999"): + with self.subTest(location=location): + document = graph_document() + document["nodes"][0]["source_location"] = location + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_a_node_outside_the_indexed_checkout(self) -> None: + """A node that could never be cited must not be traversable either.""" + for path in ("/etc/passwd", "../sibling/config.py", ".git/config", ".graphify/nodes.bin"): + with self.subTest(path=path): + document = graph_document() + document["nodes"][0]["source_file"] = path + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_duplicate_node_identifiers(self) -> None: + document = graph_document() + document["nodes"].append(dict(document["nodes"][0])) + with self.assertRaises(ContextError): + self.load(document) + + +class EdgeLocationTests(unittest.TestCase): + """The call site's own path and line, held to the same rules as a node's. + + ``source_file`` is a required provider field and every fixture edge carries + a real one; what varies here is ``source_location``, which is optional the + same way a node's is, and the path itself, which is held to the same + repository-scope rules ``test_refuses_a_node_outside_the_indexed_checkout`` + holds a node's ``source_file`` to. + """ + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def touched(self, graph: query.CodeGraph) -> query.GraphEdge: + """The one edge these tests mutate: ``n-load`` calls ``n-config``.""" + return next(item for item in graph.edges + if item.source == "n-load" and item.target == "n-config") + + def test_missing_null_and_empty_locations_stay_traversable_and_uncitable(self) -> None: + for mutate in ( + lambda doc: doc["edges"][0].pop("source_location"), + lambda doc: doc["edges"][0].update(source_location=None), + lambda doc: doc["edges"][0].update(source_location=""), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + edge = self.touched(self.load(document)) + self.assertIsNone(edge.line) + # Still traversable, and its citation names the file alone -- + # never a line the provider did not state. + self.assertEqual(edge.citation, "example_pkg/config.py") + + def test_a_malformed_location_type_is_refused(self) -> None: + for value in ([], {}, ["L1"], 12, True): + with self.subTest(value=value): + document = graph_document() + document["edges"][0]["source_location"] = value + with self.assertRaises(ContextError): + self.load(document) + + def test_non_ascii_digits_are_refused_rather_than_crashing(self) -> None: + """``str.isdigit()`` accepts a superscript two; ``int()`` cannot parse it. + + Before this correction ``L²`` escaped as an unhandled + ``ValueError`` instead of the bounded ``ContextError`` every other + unsupported location shape raises. + """ + document = graph_document() + document["edges"][0]["source_location"] = "L²" + with self.assertRaises(ContextError): + self.load(document) + + def test_invalid_line_numbers_are_refused(self) -> None: + for location in ("12", "line 12", "L", "L0", "L-4", "L99999999999"): + with self.subTest(location=location): + document = graph_document() + document["edges"][0]["source_location"] = location + with self.assertRaises(ContextError): + self.load(document) + + def test_an_absolute_traversing_or_private_call_site_path_is_refused(self) -> None: + for path in ("/etc/passwd", "../sibling/config.py", ".git/config", ".graphify/nodes.bin"): + with self.subTest(path=path): + document = graph_document() + document["edges"][0]["source_file"] = path + with self.assertRaises(ContextError): + self.load(document) + + +class NodeLinkFormatTests(unittest.TestCase): + """The other document that can appear under ``graph.json``, read on its own terms. + + ``export.py::to_json`` is the clustered path's writer, and its provenance is + not the extractor's: the endpoints it writes came out of a NetworkX + container that may have been undirected. So it is read, but only when it + says it preserved direction -- and a raw extraction is never held to that, + because its producer writes no such marker. + """ + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_reads_a_directed_node_link_export(self) -> None: + graph = self.load(node_link_document()) + self.assertEqual(len(graph.nodes), 5) + self.assertEqual(len(graph.edges), 4) + # The supported node-link document carries the same edge call sites as + # the raw extraction, read by the same ``_edge`` logic. + contains = next(item for item in graph.edges if item.relation == "contains") + self.assertEqual((contains.path, contains.line), ("example_pkg/config.py", 12)) + + def test_reads_the_renamed_edges_key_of_a_node_link_export(self) -> None: + """NetworkX renamed ``links`` to ``edges``; the pinned validator takes either. + + The marker keys are what identify the format, so the renamed document is + still node-link and is still held to the direction requirement. + """ + document = node_link_document() + document["edges"] = document.pop("links") + self.assertEqual(len(self.load(document).edges), 4) + undirected = node_link_document(directed=False) + undirected["edges"] = undirected.pop("links") + with self.assertRaises(ContextError): + self.load(undirected) + + def test_refuses_a_node_link_export_that_does_not_preserve_direction(self) -> None: + """An undirected export states an endpoint pair, not a caller and callee. + + The provider's undirected storage canonicalizes endpoint order and its + export's repair leaves no mark a reader can check, so every oriented + answer here -- ``impact``, ``dependency``, and the ``calls`` sentence a + ``symbol`` neighbourhood states -- would be asserting an orientation the + document never established. The refusal names direction, so an operator + reads it as "rebuild" rather than as a corrupt graph. + """ + for flag in (False, None, "true", 1): + with self.subTest(directed=flag): + document = node_link_document() + if flag is None: + # Still node-link: ``multigraph``, ``graph`` and ``links`` + # are markers of their own, so dropping one key does not + # make this document pass as a raw extraction. + document.pop("directed") + else: + document["directed"] = flag + with self.assertRaises(ContextError) as caught: + self.load(document) + self.assertIn("direction", str(caught.exception)) + + def test_each_marker_alone_identifies_the_node_link_format(self) -> None: + for marker in query.NODE_LINK_MARKERS: + with self.subTest(marker=marker): + document = graph_document() + document[marker] = {} if marker == "graph" else False + if marker == "links": + document["links"] = document.pop("edges") + with self.assertRaises(ContextError) as caught: + self.load(document) + self.assertIn("direction", str(caught.exception)) + + def test_the_raw_format_is_what_the_reader_reports_for_the_pinned_options(self) -> None: + self.assertEqual(query._graph_format(graph_document()), query.GRAPH_FORMAT_RAW) + self.assertEqual( + query._graph_format(node_link_document()), query.GRAPH_FORMAT_NODE_LINK + ) + + +class TraversalTests(GraphWorkspace): + def query(self, **overrides) -> query.QueryResult: + arguments = {"question": "impact", "target": "parse_config"} + arguments.update(overrides) + return query.run_query(self.graph(), **arguments) + + def test_impact_walks_against_the_relationships(self) -> None: + result = self.query() + reached = {item.node.name for item in result.relations} + self.assertEqual(reached, {"load", "test_parse_config", "render"}) + self.assertFalse(result.truncated) + + def test_dependency_walks_along_them(self) -> None: + """``load`` depends on ``parse_config``; ``parse_config`` depends on nothing.""" + reached = self.query(target="load", question="dependency").relations + self.assertEqual({item.node.name for item in reached}, {"parse_config"}) + self.assertEqual(self.query(question="dependency").relations, ()) + + def test_related_tests_answers_with_tests_only(self) -> None: + result = self.query(question="related_tests") + self.assertEqual([item.node.name for item in result.relations], ["test_parse_config"]) + + def test_symbol_is_a_one_hop_neighbourhood(self) -> None: + result = self.query(question="symbol") + self.assertEqual({item.depth for item in result.relations}, {1}) + self.assertEqual({item.node.name for item in result.relations}, + {"load", "test_parse_config", "config.py"}) + + def test_traversal_is_deterministic(self) -> None: + first = [item.node.id for item in self.query().relations] + second = [item.node.id for item in query.run_query( + self.graph(), question="impact", target="parse_config").relations] + self.assertEqual(first, second) + + def test_a_second_hop_reports_the_edge_it_actually_walked(self) -> None: + """The graph says ``render`` calls ``load``, and nothing about render and parse_config.""" + reached = {item.node.name: item for item in self.query().relations} + self.assertEqual(reached["load"].origin.name, "parse_config") + self.assertEqual(reached["render"].depth, 2) + self.assertEqual(reached["render"].origin.name, "load") + # The seed the walk started from is still carried, as provenance rather + # than as a relationship anybody asserted. + self.assertEqual(reached["render"].seed.name, "parse_config") + + def test_budget_truncates_and_says_so(self) -> None: + result = self.query(node_budget=1) + self.assertEqual(len(result.relations), 1) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_more_definitions_than_the_seed_bound_is_reported_as_truncation(self) -> None: + """Seeds the bound dropped take their whole reachable neighbourhood with them.""" + document = graph_document() + for index in range(query.MAX_SEEDS + 1): + document["nodes"].append( + node(f"n-extra-{index}", "parse_config", "example_pkg/loader.py", 2)) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + result = query.run_query(graph, question="impact", target="parse_config") + self.assertEqual(len(result.seeds), query.MAX_SEEDS) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + self.assertIn("unresolved_entities", result.omissions) + + def test_depth_bounds_the_walk(self) -> None: + """``render`` is two hops from ``parse_config`` and out of a one-hop walk.""" + self.assertNotIn("render", {item.node.name for item in self.query(depth=1).relations}) + + def test_an_unresolved_target_is_reported_rather_than_guessed(self) -> None: + result = self.query(target="no_such_symbol") + self.assertFalse(result.resolved) + self.assertEqual(result.omissions, ("unresolved_entities",)) + + def test_out_of_range_budget_and_depth_are_refused(self) -> None: + for arguments in ({"node_budget": 0}, {"node_budget": query.MAX_NODE_BUDGET + 1}, + {"depth": 0}, {"depth": query.MAX_DEPTH + 1}): + with self.subTest(arguments=arguments): + with self.assertRaises(ContextError): + self.query(**arguments) + + def test_an_unsupported_question_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.query(question="everything") + + +class RetainedRelationshipTests(unittest.TestCase): + """A relationship between two already-seen nodes is still evidence. + + Expansion and reporting were once bounded by one set of node ids, so a + graph that says two things about a pair had one of them deleted while the + result still reported itself complete. These build their own graphs because + the shapes that expose it -- a cycle, a seed set that is already connected, + a walk that reconverges -- are not in the shared fixture. + + The bound is unchanged: each node is still walked through once, and what a + budget or a depth limit removes is still reported as truncation. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def stated(self, result: query.QueryResult) -> set: + """Every reported relationship as the provider's own edge record.""" + return {(item.via.source, item.via.relation, item.via.target) for item in result.relations} + + def test_a_two_way_cycle_reports_both_directions(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha") + self.assertEqual(self.stated(result), + {("n-b", "calls", "n-a"), ("n-a", "calls", "n-b")}) + # The second direction is the edge it says it is, between its own two + # endpoints -- not the seed relabelled. + self.assertEqual([(item.origin.name, item.node.name) + for item in result.relations if item.depth == 2], + [("beta", "alpha")]) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_relationships_among_path_seeds_are_still_reported(self) -> None: + """Every endpoint is a seed, so the old reader answered with nothing.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/config.py", 20), + edges=[edge("n-a", "n-b", "calls")], + ) + result = query.run_query(graph, question="dependency", target="example_pkg/config.py") + self.assertEqual({item.id for item in result.seeds}, {"n-a", "n-b"}) + self.assertEqual(self.stated(result), {("n-a", "calls", "n-b")}) + + def test_a_reconvergent_walk_keeps_both_paths_into_one_node(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + node("n-d", "delta", "example_pkg/config.py", 30), + edges=[edge("n-a", "n-b", "calls"), edge("n-a", "n-c", "calls"), + edge("n-b", "n-d", "calls"), edge("n-c", "n-d", "calls")], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(self.stated(result), { + ("n-a", "calls", "n-b"), ("n-a", "calls", "n-c"), + ("n-b", "calls", "n-d"), ("n-c", "calls", "n-d"), + }) + self.assertEqual({(item.origin.name, item.node.name) + for item in result.relations if item.depth == 2}, + {("beta", "delta"), ("gamma", "delta")}) + self.assertFalse(result.truncated) + + def test_a_self_loop_is_reported_once_and_parallel_relations_stay_distinct(self) -> None: + """Endpoints alone are not the identity; the provider's wording is part of it.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-a", "calls"), + edge("n-a", "n-b", "calls"), + edge("n-a", "n-b", "references")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + self.assertEqual(sorted(self.stated(result)), [ + ("n-a", "calls", "n-a"), ("n-a", "calls", "n-b"), ("n-a", "references", "n-b"), + ]) + + def test_a_duplicated_edge_record_is_reported_once(self) -> None: + """Reached from both sides of a ``both`` walk, or written twice: one relationship.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-a", "n-b", "calls")], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 1) + self.assertFalse(result.truncated) + + def test_a_retained_relationship_the_budget_cuts_is_reported_as_truncation(self) -> None: + """The budget still bounds the answer, and still says what it removed.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha", node_budget=1) + self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_depth_still_bounds_a_walk_that_retains_relationships(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha", depth=1) + self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) + + +class CallSiteIdentityTests(unittest.TestCase): + """Two calls to the same relationship, recorded at two places, are two. + + ``edge()`` gives every edge the same call site unless a test overrides it, + so these override it directly. This is the finding's own case: a provider + that records more than one call site for one relationship, and a reader + that dropped the call site from the identity reported one when the + document stated several. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def sites(self, result: query.QueryResult) -> set: + return {(item.via.path, item.via.line) for item in result.relations} + + def test_same_endpoints_relation_and_confidence_at_different_lines_are_distinct(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/config.py", 9)}, + ) + self.assertFalse(result.truncated) + + def test_the_same_line_in_different_files_is_distinct(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_file="example_pkg/config.py", source_location="L5"), + edge("n-a", "n-b", "calls", source_file="example_pkg/loader.py", source_location="L5"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/loader.py", 5)}, + ) + + def test_an_exact_duplicate_record_still_collapses(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L5"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 1) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_duplicate_plus_one_distinct_location_yields_two(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/config.py", 9)}, + ) + + def test_different_relations_or_confidences_at_one_call_site_stay_distinct(self) -> None: + """One call site, two different provider claims about it: both survive.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "references", source_location="L5"), + edge("n-a", "n-b", "calls", "INFERRED", source_location="L5"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + + def test_reversed_edge_input_yields_identical_identities_and_order(self) -> None: + """A permutation of the provider's own edge list must not change the answer.""" + edges = [ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + edge("n-a", "n-b", "references", source_location="L5"), + ] + forward = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=edges, + ) + reversed_graph = self.load( + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=list(reversed(edges)), + ) + def identity(item: query.GraphEdge) -> tuple: + return (item.source, item.target, item.relation, item.path, item.line) + + self.assertEqual( + [identity(item) for item in forward.edges], + [identity(item) for item in reversed_graph.edges], + ) + first = query.run_query(forward, question="dependency", target="alpha") + second = query.run_query(reversed_graph, question="dependency", target="alpha") + self.assertEqual( + [(item.via.relation, item.via.path, item.via.line) for item in first.relations], + [(item.via.relation, item.via.path, item.via.line) for item in second.relations], + ) + + def test_incoming_outgoing_and_symbol_traversal_use_the_full_identity(self) -> None: + """Two distinct call sites survive whichever direction reaches them.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + self.assertEqual(len(query.run_query(graph, question="dependency", target="alpha").relations), 2) + self.assertEqual(len(query.run_query(graph, question="impact", target="beta").relations), 2) + self.assertEqual(len(query.run_query(graph, question="symbol", target="alpha").relations), 2) + + def test_a_self_loop_at_distinct_call_sites_is_two_relationships(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=[ + edge("n-a", "n-a", "calls", source_location="L5"), + edge("n-a", "n-a", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + + def test_a_cycle_at_distinct_call_sites_reports_every_record(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-b", "n-a", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertFalse(result.truncated) + + def test_relationships_among_path_seeds_stay_one_when_the_call_site_repeats(self) -> None: + """Every endpoint is a seed and the record is not repeated: still one.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/config.py", 20), + edges=[edge("n-a", "n-b", "calls", source_location="L5")], + ) + result = query.run_query(graph, question="dependency", target="example_pkg/config.py") + self.assertEqual({item.id for item in result.seeds}, {"n-a", "n-b"}) + self.assertEqual(len(result.relations), 1) + + +class DepthBoundaryTests(unittest.TestCase): + """What the requested depth left behind, said out loud. + + Stopping at the requested depth is the contract. Stopping quietly is not: + for ``a -> b -> c -> d`` a default ``dependency`` question about ``a`` + answers with ``c`` and used to call that answer complete, so a reader + concluded ``c`` depends on nothing -- the graph's own record of ``c -> d`` + contradicting a packet that claimed to carry everything. + + The flag is not "the walk reached its depth". It is "the walk reached its + depth *and* left an eligible relationship unreported", measured the way the + walk measures eligibility: this question's direction and relationship + filter, against relationship identities not already in the answer. So these + hold both directions of that -- what must be reported, and what must not + become a false omission. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def chain(self, *, tail=()) -> query.CodeGraph: + """``alpha -> beta -> gamma``, plus whatever a test hangs off it.""" + return self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + node("n-d", "delta", "example_pkg/config.py", 30), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-c", "calls"), *tail], + ) + + def names(self, result: query.QueryResult) -> set: + return {item.node.name for item in result.relations} + + def test_a_chain_past_the_requested_depth_is_reported_as_truncation(self) -> None: + """The finding's own case: A -> B -> C -> D answered at depth 2.""" + graph = self.chain(tail=[edge("n-c", "n-d", "calls")]) + result = query.run_query(graph, question="dependency", target="alpha") + # The answer itself is unchanged -- the depth limit still bounds it. + self.assertEqual(self.names(result), {"beta", "gamma"}) + self.assertNotIn("delta", self.names(result)) + # What changed is that it no longer claims to be everything. + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + # Still not ``provider_partial``: the artifact carried ``c -> d`` in + # full. A bound of ours cut it, which is a different fact. + self.assertNotIn("provider_partial", result.omissions) + + def test_a_chain_that_ends_at_the_boundary_stays_complete(self) -> None: + """Exactly at the limit with nothing behind it: there is nothing to report.""" + result = query.run_query(self.chain(), question="dependency", target="alpha") + self.assertEqual(self.names(result), {"beta", "gamma"}) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_cycle_whose_boundary_edges_are_already_reported_stays_complete(self) -> None: + """Both records are in the answer, so the boundary omitted nothing.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_an_already_reported_self_loop_and_parallel_pair_leave_it_complete(self) -> None: + """Identity, not endpoints: every record incident to the boundary is stated.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-a", "calls"), + edge("n-a", "n-b", "calls"), + edge("n-a", "n-b", "references")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_boundary_relationship_back_into_the_answer_is_still_an_omission(self) -> None: + """Both endpoints are already reported nodes; the relationship is not. + + ``c -> a`` closes the cycle onto the seed, and its parallel twin says + something else about the same pair. Asking whether the boundary reaches + an *unseen node* would call this answer complete and drop two records + the graph carries, which is the identity confusion that produced the + earlier retained-relationship finding, one hop further out. + """ + graph = self.chain(tail=[edge("n-c", "n-a", "calls"), + edge("n-c", "n-a", "references")]) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_a_boundary_relationship_that_differs_only_by_call_site_is_still_an_omission(self) -> None: + """Same endpoints and relation, two recorded call sites: two identities.""" + graph = self.chain(tail=[edge("n-c", "n-a", "calls", source_location="L1"), + edge("n-c", "n-a", "calls", source_location="L2")]) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_direction_decides_what_the_boundary_counts(self) -> None: + """One graph, two questions: the boundary edge points the wrong way for one.""" + graph = self.chain(tail=[edge("n-d", "n-c", "calls")]) + along = query.run_query(graph, question="dependency", target="alpha") + self.assertFalse(along.truncated) + self.assertNotIn("provider_has_more", along.omissions) + both = query.run_query(graph, question="symbol", target="alpha", depth=2) + self.assertTrue(both.truncated) + self.assertIn("provider_has_more", both.omissions) + + def test_the_relationship_filter_decides_what_the_boundary_counts(self) -> None: + """``contains`` normalizes to ``defines``, which ``dependency`` never walks.""" + graph = self.chain(tail=[edge("n-c", "n-d", "contains")]) + filtered = query.run_query(graph, question="dependency", target="alpha") + self.assertFalse(filtered.truncated) + self.assertNotIn("provider_has_more", filtered.omissions) + # ``symbol`` carries the whole vocabulary, so the same edge counts there. + neighbourhood = query.run_query(graph, question="symbol", target="alpha", depth=2) + self.assertTrue(neighbourhood.truncated) + self.assertIn("provider_has_more", neighbourhood.omissions) + + def test_an_unresolved_target_reports_no_depth_omission(self) -> None: + """Nothing was traversed, so the depth limit cut nothing.""" + result = query.run_query(self.chain(), question="dependency", target="no_such_symbol") + self.assertEqual(result.omissions, ("unresolved_entities",)) + self.assertFalse(result.truncated) + + +def cyclic_graph_document() -> dict: + """The shared fixture, plus the back-edge that makes the pair mutual. + + ``load`` already calls ``parse_config``; this adds the provider's own + record that ``parse_config`` also calls ``load``. + """ + return graph_document(edges=[*graph_edges(), edge("n-config", "n-load", "calls")]) + + +class RetainedRelationshipPacketTests(GraphWorkspace): + """What a recipient actually reads for a retained relationship.""" + + document = cyclic_graph_document() + + def documents(self) -> dict: + return {item["text"]: item for item in self.context().packet["documents"]} + + def test_a_retained_back_edge_states_and_cites_its_own_endpoints(self) -> None: + documents = self.documents() + [text] = [item for item in documents if "parse_config calls load" in item] + self.assertIn("hop 2", text) + self.assertEqual( + {citation["source"] for citation in documents[text]["citations"]}, + {"example_pkg/config.py#L12", "example_pkg/loader.py#L40"}, + ) + # The other direction between the same pair is still its own document, + # stated the way the provider recorded it. + self.assertTrue(any("load calls parse_config" in item for item in documents)) + + def test_the_rest_of_the_walk_is_unchanged(self) -> None: + documents = self.documents() + self.assertTrue(any("render calls load" in item and "reached from parse_config" in item + for item in documents)) + self.assertNotIn("provider_has_more", self.context().summary["omissions"]) + + +def call_site_document(*sites: tuple, relation: str = "calls") -> dict: + """``alpha`` and ``beta``, connected once per call site in ``sites``. + + Each entry is a distinct ``(path, line)`` the same relationship was + recorded at -- the finding's own case, where a provider stating one + relationship at more than one call site had all but one silently dropped. + """ + return graph_document( + nodes=[ + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + ], + edges=[ + edge("n-a", "n-b", relation, source_file=path, source_location=f"L{line}") + for path, line in sites + ], + ) + + +class CallSitePacketTests(GraphWorkspace): + """What a recipient reads when the provider records more than one call site.""" + + document = call_site_document(("example_pkg/config.py", 5), ("example_pkg/config.py", 9)) + + def query(self, **overrides) -> query.GraphContext: + arguments = {"question": "dependency", "target": "alpha"} + arguments.update(overrides) + return self.context(**arguments) + + def test_distinct_call_sites_produce_distinguishable_documents_and_citations(self) -> None: + outcome = self.query() + self.assertEqual(outcome.status, query.AVAILABLE) + self.assertEqual(len(outcome.packet["documents"]), 2) + citations = [ + {citation["source"] for citation in item["citations"]} + for item in outcome.packet["documents"] + ] + every_source = {source for group in citations for source in group} + self.assertIn("example_pkg/config.py#L5", every_source) + self.assertIn("example_pkg/config.py#L9", every_source) + # Each document names the call site its own relationship record was + # actually written at, not the other one -- that is what makes them + # distinguishable evidence rather than the same document twice. + self.assertNotEqual(citations[0], citations[1]) + self.assertFalse(outcome.packet["truncated"]) + + def test_reversed_edge_input_produces_identical_packet_documents(self) -> None: + forward = self.query().packet["documents"] + reversed_document = call_site_document( + ("example_pkg/config.py", 9), ("example_pkg/config.py", 5)) + reversed_document["nodes"] = list(reversed(reversed_document["nodes"])) + reversed_document["edges"] = list(reversed(reversed_document["edges"])) + self.publish(reversed_document) + reversed_documents = self.query().packet["documents"] + self.assertEqual( + [item["text"] for item in forward], + [item["text"] for item in reversed_documents], + ) + self.assertEqual( + [{c["source"] for c in item["citations"]} for item in forward], + [{c["source"] for c in item["citations"]} for item in reversed_documents], + ) + + def test_a_real_second_call_site_beyond_the_relationship_budget_sets_provider_has_more(self) -> None: + outcome = self.query(node_budget=1) + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertTrue(outcome.packet["truncated"]) + self.assertIn("provider_has_more", outcome.packet["omissions"]) + + def test_a_document_budget_below_the_call_site_count_sets_document_limit(self) -> None: + outcome = self.query(policy=policy(max_documents=1)) + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertTrue(outcome.packet["truncated"]) + self.assertIn("document_limit", outcome.packet["omissions"]) + + def test_duplicate_only_input_does_not_falsely_truncate(self) -> None: + self.publish(call_site_document( + ("example_pkg/config.py", 5), ("example_pkg/config.py", 5))) + outcome = self.query() + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertFalse(outcome.packet["truncated"]) + self.assertNotIn("provider_has_more", outcome.packet["omissions"]) + + def test_an_unresolved_call_site_citation_is_omitted_and_warned(self) -> None: + """A stated call site past the end of a tracked file is never verified.""" + self.publish(call_site_document(("example_pkg/config.py", 999))) + outcome = self.query() + self.assertEqual(outcome.status, query.AVAILABLE) + [document] = outcome.packet["documents"] + sources = {citation["source"] for citation in document["citations"]} + self.assertNotIn("example_pkg/config.py#L999", sources) + self.assertEqual(sources, {"example_pkg/config.py#L12", "example_pkg/loader.py#L40"}) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + def test_an_untracked_call_site_path_is_never_cited(self) -> None: + self.publish(call_site_document(("example_pkg/untracked.py", 1))) + outcome = self.query() + [document] = outcome.packet["documents"] + sources = {citation["source"] for citation in document["citations"]} + self.assertNotIn("example_pkg/untracked.py#L1", sources) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + +def missing_endpoint_document() -> dict: + """The shared fixture, plus a relationship onto an id it never declares. + + ``parse_config`` calls something the provider named ``n-ghost`` and then + described nowhere. The relationship is real and unreadable, which is not + the same fact as a relationship onto a corpus this module has stated it + does not query. + """ + return graph_document(edges=[*graph_edges(), edge("n-config", "n-ghost", "calls")]) + + +class MissingEndpointTests(unittest.TestCase): + """A dropped edge is two different facts, and must not be reported as one. + + The provider's node list can be missing an endpoint for two reasons. It + declared the endpoint as a document, paper, image, rationale or concept, + and this module has stated in its own contract that it does not query those + -- a scope, readable in the rules. Or it declared the endpoint nowhere at + all, which is evidence its own document does not carry. Pruning both + silently let a query over a neighbourhood the provider could not state in + full come back marked ``complete``. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_a_declared_exclusion_leaves_no_node_incomplete(self) -> None: + """The stated scope stays silent, exactly as before.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["edges"].append(edge("n-config", "n-doc", "references")) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + self.assertNotIn("n-doc", graph.nodes) + self.assertEqual(graph.incomplete, frozenset()) + + def test_an_undeclared_endpoint_marks_the_surviving_node(self) -> None: + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + self.assertEqual(graph.incomplete, frozenset({"n-config"})) + # Still pruned: a relationship with one end unstated is not citable. + self.assertNotIn("n-ghost", {edge_.target for edge_ in graph.edges}) + + def test_both_ends_of_an_undeclared_relationship_are_marked(self) -> None: + """Direction is not what decides it; being in the graph is.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-ghost", "n-a", "calls"), edge("n-b", "n-ghost", "calls")], + ) + self.assertEqual(graph.incomplete, frozenset({"n-a", "n-b"})) + + def test_a_relationship_with_no_surviving_endpoint_marks_nothing(self) -> None: + """No node any traversal can reach is narrowed by it, so nothing claims it.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=[edge("n-ghost", "n-other-ghost", "calls")], + ) + self.assertEqual(graph.incomplete, frozenset()) + self.assertEqual(graph.edges, ()) + + def test_a_mixed_relationship_counts_as_missing_evidence(self) -> None: + """One declared exclusion does not excuse the end that was never declared.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["edges"].append(edge("n-config", "n-ghost", "calls")) + document["edges"].append(edge("n-load", "n-doc", "references")) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + self.assertEqual(graph.incomplete, frozenset({"n-config"})) + + def test_a_traversal_that_reaches_the_node_reports_partial(self) -> None: + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="symbol", target="parse_config") + self.assertIn("provider_partial", result.omissions) + + def test_a_missing_endpoint_is_not_itself_reported_as_truncation(self) -> None: + """No budget and no depth limit cut this; the artifact never carried it. + + The traversal is ``impact`` rather than the one-hop ``symbol`` + neighbourhood above because the two facts must not be read through each + other. In this fixture a ``symbol`` walk stops one hop out at ``load``, + which really does still have ``render calls load`` behind it -- genuine + depth truncation, and now reported as such. That would make a + ``assertFalse(truncated)`` here prove nothing about missing endpoints. + ``impact`` ends at ``render``, which has no eligible relationship left, + so the only thing this answer has left out is the endpoint the provider + never declared -- and that is partial without being truncated. + """ + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="impact", target="parse_config") + self.assertIn("provider_partial", result.omissions) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_traversal_that_reaches_the_node_indirectly_reports_partial(self) -> None: + """Touched by the walk, not just seeded: the answer still spans that node.""" + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="dependency", target="load") + self.assertTrue(any(item.node.id == "n-config" for item in result.relations)) + self.assertIn("provider_partial", result.omissions) + + def test_a_traversal_elsewhere_in_the_graph_stays_complete(self) -> None: + """A hole somewhere else is not a hole in this answer. + + Marking every query partial because one node in the repository lost an + endpoint would make the flag say nothing about the answer carrying it. + """ + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + edges=[edge("n-a", "n-b", "calls"), edge("n-c", "n-ghost", "calls")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertNotIn("provider_partial", result.omissions) + self.assertFalse(result.truncated) + + +class MissingEndpointPacketTests(GraphWorkspace): + """What the recipient reads when the provider could not state a neighbourhood.""" + + document = missing_endpoint_document() + + def test_the_packet_is_partial_and_says_why(self) -> None: + context = self.context() + self.assertIn("provider_partial", context.packet["omissions"]) + self.assertEqual(context.packet["completeness"], "partial") + # A packet may be partial without being truncated; the delivery contract + # only forbids the other pairing. + self.assertFalse(context.packet["truncated"]) + self.assertIn("provider_partial", context.summary["omissions"]) + self.assertEqual(context.summary["completeness"], "partial") + + def test_the_relationships_it_could_state_are_still_stated(self) -> None: + """Partial is not empty: what the document did carry is still evidence.""" + texts = [item["text"] for item in self.context().packet["documents"]] + self.assertTrue(any("load calls parse_config" in text for text in texts)) + + +class CallableLabelSeedTests(unittest.TestCase): + """Bare names against the labels the pinned extractor actually writes. + + A fresh contained extraction of the public fixture at 0.9.58 labels the + function ``parse_graph_citation`` as ``parse_graph_citation()`` and the + function ``packet`` as ``packet()``. An exact-label-only reader answers + "unresolved" for every natural spelling of a function name, so a bare symbol + resolves to the canonical callable label with only its syntactic trailing + argument list removed -- and to nothing else. These tests are as much about + what that must *not* reach. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_a_bare_name_resolves_to_the_canonical_callable_label(self) -> None: + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual([item.id for item in graph.seeds("parse_graph_citation")], ["n-citation"]) + + def test_the_literal_label_still_resolves_exactly(self) -> None: + """Stripping is an addition, not a replacement: the written label wins.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual( + [item.id for item in graph.seeds("parse_graph_citation()")], ["n-citation"]) + + def test_an_exact_label_beats_the_same_string_read_as_a_callable(self) -> None: + """Both exist, and the tier that matched what was written is the answer. + + A module attribute ``packet`` and a function ``packet()`` are different + definitions. Merging them would report a relationship of one as a + relationship of the other, so the exact label resolves alone and the + result is not ambiguous. + """ + graph = self.load( + node("n-attribute", "packet", "example_pkg/config.py", 4), + node("n-function", "packet()", "example_pkg/config.py", 12), + ) + self.assertEqual([item.id for item in graph.seeds("packet")], ["n-attribute"]) + + def test_a_near_name_does_not_reach_a_longer_callable(self) -> None: + """No prefix, substring, or edit-distance matching -- an equality test.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12), + node("n-other", "parse_graph_citations()", "example_pkg/config.py", 20), + ) + for target in ("parse_graph", "parse", "graph_citation", "arse_graph_citation"): + with self.subTest(target=target): + self.assertEqual(graph.seeds(target), ()) + + def test_overload_like_labels_are_ambiguity_rather_than_a_pick(self) -> None: + """Two labels reduce to one name, so the target names two definitions.""" + graph = self.load( + node("n-int", "render(int)", "example_pkg/report.py", 5), + node("n-str", "render(str)", "example_pkg/report.py", 9), + ) + result = query.run_query(graph, question="symbol", target="render") + self.assertEqual([item.id for item in result.seeds], ["n-int", "n-str"]) + self.assertTrue(result.ambiguous) + self.assertIn("unresolved_entities", result.omissions) + + def test_the_seed_bound_and_its_truncation_still_apply_to_bare_names(self) -> None: + nodes = [ + node(f"n-{index:02d}", f"render(arg{index})", "example_pkg/report.py", index + 1) + for index in range(query.MAX_SEEDS + 1) + ] + result = query.run_query(self.load(*nodes), question="symbol", target="render") + self.assertEqual(len(result.seeds), query.MAX_SEEDS) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_a_path_target_is_unchanged_and_still_last(self) -> None: + """Paths are literal, and a callable label never stands in for one.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual( + [item.id for item in graph.seeds("example_pkg/config.py")], ["n-citation"]) + self.assertEqual(graph.seeds("example_pkg"), ()) + + def test_only_a_whole_trailing_argument_list_is_removed(self) -> None: + """The rule as a table, including every shape that must reduce to nothing.""" + for label, expected in ( + ("parse_graph_citation()", "parse_graph_citation"), + ("render(int)", "render"), + ("render(Callable[(int)])", "render"), + ("parse_graph_citation", ""), + ("()", ""), + ("render(int))", ""), + ("render()x", ""), + ("render(int", ""), + ("ren)der()", ""), + ("", ""), + ): + with self.subTest(label=label): + self.assertEqual(query._callable_base(label), expected) + + +class CitationValidationTests(GraphWorkspace): + def validator(self) -> query.CitationValidator: + census = lifecycle.read_tracked_census(self.repository, self.manifest.commit) + return query.CitationValidator(self.repository, census) + + def test_validates_line_claims_against_the_bound_commit(self) -> None: + validator = self.validator() + self.assertTrue(validator.validate("example_pkg/config.py#L12")) + self.assertTrue(validator.validate("example_pkg/config.py#L40")) + self.assertFalse(validator.validate("example_pkg/config.py#L41")) + + def test_an_untracked_path_is_never_cited(self) -> None: + (self.repository / "example_pkg" / "scratch.py").write_text("x\n", encoding="utf-8") + self.assertFalse(self.validator().validate("example_pkg/scratch.py#L1")) + + def test_working_tree_edits_do_not_change_the_verdict(self) -> None: + """The generation binds a commit; the file on disk today is not evidence.""" + (self.repository / "example_pkg" / "config.py").write_text("one line\n", encoding="utf-8") + self.assertTrue(self.validator().validate("example_pkg/config.py#L40")) + + def test_an_out_of_scope_citation_is_refused(self) -> None: + for source in ("../escape.py", "/etc/passwd", ".git/config"): + with self.subTest(source=source): + self.assertFalse(self.validator().validate(source)) + + +class PacketTests(GraphWorkspace): + def test_a_packet_binds_the_generation_its_evidence_actually_came_from(self) -> None: + """An envelope authorizing another generation is refused, not copied. + + The delivery contract's freshness rules all read the binding's + generation, so a packet that carried an authorized-but-unqueried + generation would keep passing them after a rebuild -- the one case the + rules exist to catch. Nothing here rewrites the envelope: it is an + authorization this module did not mint. + """ + republished = self.publish(graph_document()) + self.assertNotEqual(republished.generation, self.manifest.generation) + # An authorization minted before the rebuild: it names the generation + # that is gone, while the traversal reads the one published now. + outcome = self.context(envelope=self.authorized(generation=self.manifest.generation)) + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "uncitable") + self.assertIsNone(outcome.packet) + + def test_packet_carries_its_provenance_and_loads_through_the_contract(self) -> None: + outcome = self.context() + self.assertEqual(outcome.status, query.AVAILABLE) + packet = outcome.packet + self.assertEqual(packet["kind"], "repository") + self.assertEqual(packet["source_revision"], self.manifest.commit) + self.assertEqual(packet["source_built_at"], self.manifest.built_at) + self.assertEqual(outcome.summary["graph_generation"], self.manifest.generation) + validated = self.load(packet, recipient="claude:builder", revision=self.manifest.commit) + self.assertEqual(validated.revision_state, "matching") + + def test_every_citation_resolves_against_the_immutable_tree(self) -> None: + outcome = self.context() + report = context_graph.evaluate_graph_evidence( + outcome.packet, repository_root=self.repository, revision_state="matching", + ) + self.assertEqual(report.resolution_rate, 1.0) + self.assertTrue(report.meets_gate()) + + def test_a_citation_past_the_end_of_a_file_is_dropped_not_delivered(self) -> None: + """Evidence that cannot be pointed at is not weaker evidence; it is none.""" + document = graph_document() + document["nodes"][1]["source_location"] = "L400" + self.publish(document) + outcome = self.context() + cited = {citation["source"] + for item in outcome.packet["documents"] for citation in item["citations"]} + self.assertNotIn("example_pkg/loader.py#L400", cited) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + def test_confidence_maps_extracted_inferred_and_ambiguous(self) -> None: + document = graph_document() + document["edges"][2]["confidence"] = "AMBIGUOUS" + self.publish(document) + outcome = self.context() + confidences = {item["confidence"] for item in outcome.packet["documents"]} + self.assertEqual(confidences, {"extracted", "inferred", "unknown"}) + self.assertIn("unresolved_entities", outcome.packet["omissions"]) + + def test_truncation_is_reported_rather_than_hidden(self) -> None: + outcome = self.context(node_budget=1) + self.assertTrue(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "partial") + self.assertIn("provider_has_more", outcome.packet["omissions"]) + + def test_a_partial_generation_is_carried_into_the_packet(self) -> None: + """A partial build's answer is partial however complete the traversal was.""" + self.publish(graph_document(), completeness=lifecycle.PARTIAL) + outcome = self.context(policy=policy(required=False)) + # A partial generation is not usable at all by default, so the optional + # request degrades rather than delivering an answer that looks whole. + self.assertEqual(outcome.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "partial") + + def test_multi_hop_evidence_names_and_cites_the_edge_it_walked(self) -> None: + """A transitive result reads as a path, never as a direct relationship.""" + documents = {item["text"]: item for item in self.context().packet["documents"]} + [text] = [item for item in documents if "render" in item] + self.assertIn("render calls load", text) + self.assertIn("reached from parse_config", text) + # The two endpoints, plus the edge's own call site -- ``edge()``'s own + # location, distinct from both endpoints in this fixture. + self.assertEqual( + {citation["source"] for citation in documents[text]["citations"]}, + {"example_pkg/report.py#L5", "example_pkg/loader.py#L40", "example_pkg/config.py#L12"}, + ) + # Each endpoint citation is titled with the node it points at, not with + # the node the relationship happened to reach; the call site is titled + # apart from either. + self.assertEqual( + {citation["title"] for citation in documents[text]["citations"]}, + {"symbol render", "symbol load", "call site: calls"}, + ) + + def test_packet_text_carries_no_indexed_content(self) -> None: + outcome = self.context() + prose = " ".join(item["text"] for item in outcome.packet["documents"]) + self.assertNotIn("line 12", prose) + for name in ("parse_config", "impact"): + self.assertIn(name, prose) + + +class DocumentBudgetTests(GraphWorkspace): + """The selected policy's document budget is what the packet is built to. + + ``MAX_DOCUMENTS`` is this adapter's own ceiling, and the shared contract + carries a separate, smaller default. A packet built to the ceiling alone is + not merely generous: ``context_contract`` refuses it at delivery, so the + answer a wide question deserves -- five documents and an honest + ``document_limit`` -- is instead no answer at all. These tests hold the two + budgets together and in the right direction: a policy may tighten what one + packet carries, never lift the ceiling. + """ + + def wide(self, callers: int = 6, **overrides) -> dict: + self.publish(wide_graph_document(callers)) + outcome = self.context(**overrides) + self.assertEqual(outcome.status, query.AVAILABLE) + return outcome.packet + + def test_the_default_policy_budget_bounds_a_wider_answer(self) -> None: + packet = self.wide() + self.assertEqual(len(packet["documents"]), 5) + self.assertLessEqual(len(packet["documents"]), contract.normalize_policy(policy())["max_documents"]) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_the_bounded_packet_is_the_deterministic_prefix_of_the_whole_answer(self) -> None: + """Nothing is reordered to fit: the budget cuts the tail, in place.""" + whole = self.wide(policy=policy(max_documents=6)) + bounded = self.wide(policy=policy(max_documents=5)) + self.assertEqual(len(whole["documents"]), 6) + self.assertEqual( + [item["text"] for item in bounded["documents"]], + [item["text"] for item in whole["documents"]][:5], + ) + + def test_an_answer_that_exactly_fits_its_budget_is_not_called_truncated(self) -> None: + """No eligible relationship was left out, so there is nothing to report.""" + packet = self.wide(policy=policy(max_documents=6)) + self.assertEqual(len(packet["documents"]), 6) + self.assertFalse(packet["truncated"]) + self.assertEqual(packet["completeness"], "complete") + self.assertNotIn("document_limit", packet["omissions"]) + + def test_a_budget_below_the_shared_default_bounds_the_packet_further(self) -> None: + """A policy may tighten past the contract's default, and is obeyed.""" + packet = self.wide(policy=policy(max_documents=2)) + self.assertEqual(len(packet["documents"]), 2) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_a_budget_above_the_adapters_ceiling_does_not_lift_it(self) -> None: + """The policy bounds the packet downward only; the ceiling still holds.""" + self.assertGreater(20, query.MAX_DOCUMENTS) + packet = self.wide(callers=20, policy=policy(max_documents=20)) + self.assertEqual(len(packet["documents"]), query.MAX_DOCUMENTS) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_the_summary_reports_the_same_bounded_count_as_the_packet(self) -> None: + """A metadata-only reader must not be told the answer was whole.""" + self.publish(wide_graph_document(6)) + outcome = self.context() + self.assertEqual(outcome.summary["documents"], len(outcome.packet["documents"])) + self.assertEqual(outcome.summary["documents"], 5) + self.assertTrue(outcome.summary["truncated"]) + self.assertEqual(outcome.summary["completeness"], "partial") + self.assertIn("document_limit", outcome.summary["omissions"]) + + +class DepthBoundaryPacketTests(GraphWorkspace): + """What a recipient reads when the depth limit left evidence behind. + + The traversal flag is only worth anything if it survives into the packet + and the metadata-only summary, and if a reader can tell *which* bound + spoke. A document budget and a depth limit are both "there is more", and + they are both true here at different times, so these hold them apart: the + depth case carries ``provider_has_more`` with no ``document_limit``, and + the evidence it did deliver is still fully cited. + """ + + def test_a_depth_bounded_answer_says_so_in_the_packet_and_the_summary(self) -> None: + # ``symbol`` is a one-hop neighbourhood of ``parse_config``, and the + # fixture puts ``render calls load`` one hop further out. + outcome = self.context(question="symbol") + self.assertEqual(outcome.status, query.AVAILABLE) + self.assertTrue(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "partial") + self.assertIn("provider_has_more", outcome.packet["omissions"]) + self.assertTrue(outcome.summary["truncated"]) + self.assertEqual(outcome.summary["completeness"], "partial") + self.assertIn("provider_has_more", outcome.summary["omissions"]) + + def test_the_depth_omission_is_not_reported_as_a_document_budget(self) -> None: + """Three documents against a budget of five: nothing was dropped to fit.""" + outcome = self.context(question="symbol") + self.assertEqual(len(outcome.packet["documents"]), 3) + self.assertLess( + len(outcome.packet["documents"]), + contract.normalize_policy(policy())["max_documents"], + ) + for code in ("document_limit", "provider_warning", "provider_partial"): + with self.subTest(code=code): + self.assertNotIn(code, outcome.packet["omissions"]) + + def test_the_retained_evidence_is_still_fully_cited_and_deliverable(self) -> None: + """Truncated is a statement about what is missing, not about what is there.""" + outcome = self.context(question="symbol") + report = context_graph.evaluate_graph_evidence( + outcome.packet, repository_root=self.repository, revision_state="matching", + ) + self.assertEqual(report.resolution_rate, 1.0) + self.assertTrue(report.meets_gate()) + validated = self.load( + outcome.packet, recipient="claude:builder", revision=self.manifest.commit) + self.assertEqual(validated.revision_state, "matching") + + def test_a_question_whose_walk_runs_out_first_is_still_complete(self) -> None: + """The same generation, a question that reaches the end of its evidence. + + ``impact`` stops at ``render``, which nothing calls, so its boundary + has no eligible relationship behind it. Without this the depth flag + could be satisfied by marking every answer partial, which would say + nothing at all. + """ + outcome = self.context() + self.assertFalse(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "complete") + self.assertNotIn("provider_has_more", outcome.packet["omissions"]) + + +class RecipientNeutralityTests(GraphWorkspace): + """One approved packet, three recipients, no provider tools or credentials.""" + + #: A packet identity, not a secret: ``render_evidence`` requires a handle. + DELIVERY = "0123456789abcdef0123456789abcdef" + + def test_claude_codex_and_devin_receive_the_same_packet(self) -> None: + packet = self.context().packet + rendered = set() + for recipient in ("claude:builder", "codex:reviewer", "devin:builder"): + with self.subTest(recipient=recipient): + validated = self.load(packet, recipient=recipient, revision=self.manifest.commit) + self.assertEqual(validated.shareable_summary()["kind"], "repository") + rendered.add(context_delivery.render_evidence(validated, self.DELIVERY)) + # The delivered payload is identical for every recipient: nothing in it + # names a connection, a credential, a provider tool, or a local path. + self.assertEqual(len(rendered), 1) + payload = rendered.pop() + self.assertNotIn("graphify", payload.casefold()) + self.assertNotIn(str(self.repository), payload) + + def test_an_unauthorized_recipient_is_still_refused(self) -> None: + with self.assertRaises(ContextError): + self.load(self.context().packet, recipient="claude:orchestrator", + revision=self.manifest.commit) + + +class AvailabilityTests(GraphWorkspace): + """Required unavailable context blocks; optional unavailable context degrades.""" + + def move_head(self) -> None: + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + + def test_stale_graph_blocks_required_context(self) -> None: + self.move_head() + outcome = self.context() + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.dependent_work, "paused") + self.assertEqual(outcome.exit_code, 1) + self.assertEqual(outcome.summary["reason"], "stale") + self.assertIsNone(outcome.packet) + + def test_stale_graph_degrades_optional_context_to_ordinary_tools(self) -> None: + self.move_head() + outcome = self.context(policy=policy(required=False)) + self.assertEqual(outcome.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(outcome.dependent_work, "usable") + self.assertEqual(outcome.exit_code, 0) + self.assertIn("ordinary repository tools", outcome.summary["next_action"]) + + def test_an_absent_graph_is_not_a_failure_of_the_caller(self) -> None: + lifecycle.remove_graph(self.repository, root=self.state) + self.assertEqual(self.context(policy=policy(required=False)).status, + query.OPTIONAL_UNAVAILABLE) + self.assertEqual(self.context().status, query.REQUIRED_UNAVAILABLE) + + def test_a_generation_without_a_provider_export_is_unreadable(self) -> None: + """A member that is not the pinned exporter's document at all.""" + self.publish({"schema": "graphify.native.v1", "entities": [], "relations": []}) + outcome = self.context() + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "unreadable") + + def test_a_malformed_vocabulary_field_reports_unreadable_both_ways(self) -> None: + """Required blocks and optional degrades, which is what "unreadable" means. + + The reader refuses a ``file_type`` that is a JSON array rather than a + word, and this is the path that refusal has to arrive on: a graph the + caller is told it cannot use, not an exception out of an opt-in + feature. Both dispositions are covered because only one of them pauses + the dependent work. + """ + document = graph_document() + document["nodes"][0]["file_type"] = ["code"] + self.publish(document) + blocked = self.context() + self.assertEqual(blocked.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(blocked.summary["reason"], "unreadable") + self.assertEqual(blocked.dependent_work, "paused") + degraded = self.context(policy=policy(required=False)) + self.assertEqual(degraded.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(degraded.summary["reason"], "unreadable") + self.assertEqual(degraded.dependent_work, "usable") + + def test_an_unresolved_target_blocks_required_context(self) -> None: + outcome = self.context(target="no_such_symbol") + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "unresolved") + + def test_summaries_carry_metadata_only(self) -> None: + for outcome in (self.context(), self.context(target="no_such_symbol")): + rendered = json.dumps(outcome.summary) + self.assertNotIn(str(self.repository), rendered) + self.assertNotIn(str(self.state), rendered) + + +class CommandTests(GraphWorkspace): + def authorization(self, **overrides) -> Path: + # The command has no injected clock: it authorizes against the real one, + # exactly as an operator's run does. So the envelope has to be live now + # rather than at the fixed ``NOW`` the library-level tests use. + live = datetime.now(timezone.utc) + timedelta(minutes=30) + payload = { + "connection": self.authorized(expires_at=live.isoformat()), + "policy": policy(), + "repository": "owner/repo", + "work_item": "work-item-one", + } + payload.update(overrides) + path = self.root / "authorization.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def invoke(self, *arguments: str) -> tuple[int, dict]: + stream = io.StringIO() + with contextlib.redirect_stdout(stream): + code = command.main([ + "query", "--repo-path", str(self.repository), "--state-dir", str(self.state), + "--authorization", str(self.authorization()), "--json", *arguments, + ]) + return code, json.loads(stream.getvalue()) + + def test_query_emits_a_metadata_summary_and_writes_a_private_packet(self) -> None: + destination = self.root / "packet.json" + code, summary = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 0) + self.assertEqual(summary["status"], query.AVAILABLE) + self.assertEqual(summary["source_revision"], self.manifest.commit) + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(destination.read_text())["kind"], "repository") + # The summary is what an operator may paste anywhere; the evidence is + # only in the private file they named. + self.assertNotIn("citations", json.dumps(summary)) + + def test_an_existing_readable_destination_is_replaced_by_a_private_file(self) -> None: + """A creation mode binds only a file the open created; this one replaces.""" + destination = self.root / "packet.json" + destination.write_text("stale", encoding="utf-8") + destination.chmod(0o644) + code, _ = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 0) + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(destination.read_text())["kind"], "repository") + # Nothing is left behind under a name the operator did not ask for. + self.assertEqual([item.name for item in self.root.iterdir() if "partial" in item.name], []) + + def test_required_unavailable_exits_non_zero_without_a_packet(self) -> None: + lifecycle.remove_graph(self.repository, root=self.state) + destination = self.root / "packet.json" + code, summary = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 1) + self.assertEqual(summary["status"], query.REQUIRED_UNAVAILABLE) + self.assertFalse(destination.exists()) + + def test_an_unreadable_authorization_file_fails_closed(self) -> None: + path = self.root / "authorization.json" + path.write_text(json.dumps({"connection": envelope()}), encoding="utf-8") + code = command.main([ + "query", "--repo-path", str(self.repository), "--state-dir", str(self.state), + "--authorization", str(path), "--question", "impact", "--target", "parse_config", + ]) + self.assertEqual(code, 1) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/tests/test_context_guided.py b/tests/test_context_guided.py index 5dc7bb04..85d6054f 100644 --- a/tests/test_context_guided.py +++ b/tests/test_context_guided.py @@ -6,15 +6,22 @@ import os import tempfile import unittest +import uuid from pathlib import Path from unittest import mock -from code_mower import context_audit, context_guided, context_session +from code_mower import context_audit, context_guided, context_prepare, context_session +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_contract import ContextError -from code_mower.context_delivery import deliver, read_binding, save_feedback +from code_mower.context_delivery import deliver, read_binding, retire_attachment, save_feedback +from code_mower.context_packets import _index from code_mower.context_review import INPUT_HEADER from code_mower.context_store import ContextStore import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault +from test_context_graph_connection import POLICY as GRAPH_POLICY, RECIPIENTS as GRAPH_RECIPIENTS @unittest.skipUnless(os.name == "posix", "private storage needs POSIX") @@ -157,6 +164,36 @@ def test_builder_delivery_and_review_feedback_need_no_private_request_or_revisio context_session.read(self.associations, self.session["id"])["stage"], "reviewed" ) + def test_published_delivery_from_a_non_git_directory_succeeds_for_organization_evidence(self): + """Organization evidence never depends on a code revision (issue #982).""" + self.attach() + saved = context_session.read(self.associations, self.session["id"]) + with self.patches()[0], self.patches()[1], self.patches()[2]: + evidence = context_guided.deliver_session( + self.associations, self.fixture.store, saved, repo_path=self.root, + backend=self.fixture.backend, + ) + self.assertIn("Private evidence", evidence) + self.assertEqual( + context_session.read(self.associations, self.session["id"])["context_state"], "ready", + ) + + def test_published_delivery_still_refuses_a_moved_remote_head(self): + """The existing bound-packet/current-PR-head validation is untouched (issue #982).""" + self.attach() + saved = context_session.read(self.associations, self.session["id"]) + self.head = "d" * 40 + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.fixture.store, saved, repo_path=self.root, + backend=self.fixture.backend, + ) + self.assertEqual( + context_session.read(self.associations, self.session["id"])["context_state"], + "unavailable", + ) + def test_devin_host_builder_and_reviewer_use_the_common_packet(self): self.session.update(host="devin", orchestrator="devin", participants=[{"id": "claude"}, {"id": "devin"}]) @@ -291,6 +328,13 @@ def test_process_crash_after_status_reuses_the_pending_revision(self): self.assertEqual(context_session.read(self.associations, self.session["id"])["revision"], revision) def test_process_crashes_around_local_reservation_and_publish_mark_resume(self): + # A crash inside ``reserve_attachment`` itself leaves the durable + # ``reserving`` marker behind with no binding ever created -- a + # process stop before reservation creates any binding + # (codex:1a4bc7b34687da726908). A later attach must recognize that + # state, safely finish cleanup, and only then let a fresh attach + # succeed; it must never silently resume toward publication in the + # same call that recognized the stale marker. with self.patches()[0], self.patches()[1], self.patches()[2], self.patches()[3], self.patches()[4], \ mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()): with self.assertRaises(KeyboardInterrupt): @@ -302,12 +346,21 @@ def test_process_crashes_around_local_reservation_and_publish_mark_resume(self): pr=42, backend=self.fixture.backend, ) - pending = context_session.read(self.associations, self.session["id"]) - revision = pending["revision"] - self.assertEqual(pending["attachment_state"], "pending") + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + report, code = self.attach() self.assertEqual((report["status"], code), ("attached", 0)) - self.assertEqual(context_session.read(self.associations, self.session["id"])["revision"], revision) # Move to a new head so the same session creates another intent. GitHub # accepts its comment and the binding is enabled, then the process stops @@ -371,6 +424,321 @@ def test_status_closes_pending_and_uncertain_recovery_states(self): for private in (saved["revision"], saved["packet"], "EXAMPLE-1", "owner/repo"): self.assertNotIn(private, encoded) + def test_stale_pending_intent_retires_before_reauthorizing_its_own_evidence(self): + """When the trusted remote head has moved beyond a saved pending + intent, the old identity must be retired before the state machine + ever calls ``reserve_attachment`` again for that stale evidence + (codex:1a4bc7b34687da726908).""" + self.fail_comment = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + self.attach() + pending = context_session.read(self.associations, self.session["id"]) + old_revision = pending["revision"] + old_head = pending["head"] + self.head = "d" * 40 + + real_reserve = context_guided.reserve_attachment + + def spying_reserve(*args, **kwargs): + if kwargs.get("head") == old_head: + raise AssertionError("stale evidence was reauthorized before retirement") + return real_reserve(*args, **kwargs) + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=spying_reserve), + ): + current = context_session.read(self.associations, self.session["id"]) + report, code = context_guided.attach_session( + self.associations, self.fixture.store, current, repo_path=self.root, pr=42, + backend=self.fixture.backend, + ) + self.assertEqual((report["status"], code), ("attached", 0)) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + self.assertNotEqual( + context_session.read(self.associations, self.session["id"])["revision"], old_revision, + ) + + def test_interrupted_association_update_recovers_after_stale_retirement(self): + """A crash between retiring a stale published binding and updating + the session association must be retry-safe: the next call finishes + the same identity-checked cleanup rather than reading the now-missing + binding as current evidence (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + old_revision = published["revision"] + retire_attachment( + self.fixture.store, published["connection"], published["packet"], old_revision, + ) + self.head = "d" * 40 + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + saved = context_session.read(self.associations, self.session["id"]) + self.assertNotEqual(saved["revision"], old_revision) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + self.assertEqual(len(self.comments), 2) + + def test_interrupted_index_write_during_retirement_completes_on_retry(self): + """A crash between the index write and the artifact delete inside + delivery cleanup must complete on the next attach rather than + report an inconsistent index (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + old_revision = published["revision"] + with self.fixture.store.locked(published["connection"]) as locked: + index_file, index = _index(locked) + entry = next(item for item in index["entries"] if item["handle"] == published["packet"]) + entry["deliveries"].remove(old_revision) + index_file.write(index) + self.head = "d" * 40 + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + saved = context_session.read(self.associations, self.session["id"]) + self.assertNotEqual(saved["revision"], old_revision) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + + def test_malformed_remote_head_cannot_retire_or_clear_a_published_binding(self): + """Missing, null, wrong-type, shortened, and non-hex ``head.sha`` + values are not authoritative head movement and must never retire or + clear an existing binding or session (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + missing = object() + for malformed in (missing, None, 123, ["d" * 40], "d" * 39, "d" * 41, "g" * 40, "D" * 40): + with self.subTest(head=malformed): + def _pull(*_a, _sha=malformed, **_k): + return {"head": {}} if _sha is missing else {"head": {"sha": _sha}} + with self.patches()[0], mock.patch.object( + context_guided, "fetch_pull_request", side_effect=_pull, + ), self.patches()[2], self.patches()[3], self.patches()[4]: + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.fixture.store, published, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, published) + self.assertTrue(read_binding(self.fixture.store, published["revision"])["published"]) + + def test_malformed_remote_head_cannot_disturb_a_pending_intent(self): + self.fail_comment = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + self.attach() + pending = context_session.read(self.associations, self.session["id"]) + with self.patches()[0], mock.patch.object( + context_guided, "fetch_pull_request", side_effect=lambda *_a, **_k: {"head": {"sha": None}}, + ), self.patches()[2], self.patches()[3], self.patches()[4]: + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.fixture.store, pending, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, pending) + + def test_concurrent_reconciliation_is_not_overwritten_by_the_rollback(self): + """If another operation already reconciled the exact freshly minted + intent before the rollback runs, the rollback must not overwrite that + result or claim it cleared anything (codex:1a4bc7b34687da726908).""" + real_record_failure = context_session.record_failure + + def racing_record_failure(store, record, error): + updated = real_record_failure(store, record, error) + # A concurrent operation reconciles the exact same intent first. + return context_session.update( + store, record["session_id"], expected_generation=updated["generation"], + changes={ + "stage": "prepared", "pr": None, "head": None, "revision": None, + "attachment_state": "none", + }, + ) + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_session, "record_failure", side_effect=racing_record_failure), + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + after = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (after["attachment_state"], after["pr"], after["head"], after["revision"]), + ("none", None, None, None), + ) + + def test_rollback_failure_retains_the_reserving_intent_rather_than_claiming_success(self): + """If the abandon step itself fails, the saved durable pre-publication + (``reserving``) intent is left in place rather than reported as + cleared -- a ``ContextError`` there does not prove no local write + occurred (codex:1a4bc7b34687da726908).""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + mock.patch.object(context_guided, "abandon_attachment", side_effect=ContextError("cleanup failed")), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + def test_interrupted_association_clear_after_reservation_recovers_on_retry(self): + """A crash between abandoning an unpublished binding and clearing the + session association leaves the durable ``reserving`` marker behind, + with the reservation it once named already gone. The next attach + recognizes that state, finishes cleanup idempotently, and only then + permits an explicit refresh and a successful reattachment + (codex:1a4bc7b34687da726908).""" + revision = uuid.uuid4().hex + reserving = context_session.update( + self.associations, self.session["id"], expected_generation=self.record["generation"], + changes={ + "stage": "prepared", "pr": 42, "head": self.head, "revision": revision, + "attachment_state": "reserving", + }, + ) + context_guided.reserve_attachment( + self.fixture.store, reserving["connection"], reserving["packet"], reserving["policy"], + context_guided.ContextRequest(reserving["repo"], reserving["work_item"], "codex:orchestrator"), + pr=42, head=self.head, revision=revision, backend=self.fixture.backend, + ) + # The reservation genuinely exists on disk; a crash then lands + # between the two writes ``_abandon_and_clear`` itself makes. + context_guided.abandon_attachment( + self.fixture.store, reserving["connection"], reserving["packet"], revision, + ) + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + + def test_concurrent_failure_after_cleanup_read_preserves_the_diagnostic(self): + """A ``record_failure`` that lands after ``_abandon_and_clear`` reads + the association but before it clears it -- bumping the generation + while keeping the same revision -- must not be lost: the same + revision still clears safely on a later attach while the latest + failure diagnostic survives (codex:1a4bc7b34687da726908).""" + calls = {"n": 0} + real_read = context_session.read + + def racing_read(store, session_id): + calls["n"] += 1 + current = real_read(store, session_id) + if calls["n"] == 2: + context_session.update( + store, session_id, expected_generation=current["generation"], + changes={"context_state": "authorization_failed"}, + ) + return current + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + mock.patch.object(context_session, "read", side_effect=racing_read), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + stranded = context_session.read(self.associations, self.session["id"]) + self.assertEqual(stranded["attachment_state"], "reserving") + self.assertEqual(stranded["context_state"], "authorization_failed") + revision = stranded["revision"] + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + # The failure diagnostic recorded during the race is preserved + # rather than silently erased by the eventual cleanup. + self.assertEqual(recovered["context_state"], "authorization_failed") + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + + def test_abandon_and_clear_never_touches_a_different_or_published_revision(self): + """The identity check the reserving cleanup relies on must never + clear a different, replaced, or already-published revision, even + under a stale generation (codex:1a4bc7b34687da726908).""" + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + published = context_session.read(self.associations, self.session["id"]) + stale = {**published, "revision": "0" * 32} + context_guided._abandon_and_clear(self.associations, self.fixture.store, stale) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, published) + + def test_interrupted_pending_transition_recovers_without_public_writes(self): + """A crash after a fresh reservation succeeds but before its session + association durably transitions to ``pending`` -- still strictly + before any GitHub write -- must recover on retry by abandoning the + unpublished binding and restoring prepared/none, with zero public + writes, rather than resuming toward publication in that same + recovery (codex:1a4bc7b34687da726908).""" + real_update = context_session.update + + def failing_transition(store, session_id, *, expected_generation, changes): + if changes == {"attachment_state": "pending"}: + raise OSError("disk full") + return real_update(store, session_id, expected_generation=expected_generation, changes=changes) + + with mock.patch.object(context_session, "update", side_effect=failing_transition): + # The mocked ``OSError`` never reaches the store boundary that + # would normally raise it, but it still propagates out through + # the surrounding ``association_store.locked`` block that + # ``attach_session`` holds for its whole body, so it is normalized + # to the same fail-closed public ``ContextError`` a genuine + # storage fault there would produce. + with self.assertRaisesRegex(ContextError, "private context store is unavailable or unsafe"): + self.attach() + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + revision = reserving["revision"] + # The reservation itself genuinely completed before the interrupted + # transition -- a real, unpublished binding exists. + self.assertFalse(read_binding(self.fixture.store, revision)["published"]) + self.assertEqual(self.comments, []) + self.assertEqual(self.events, []) + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + self.assertEqual(self.events, []) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + def test_provider_free_cross_process_qualification_is_symmetric_for_both_hosts(self): delivered = [] @@ -461,5 +829,324 @@ def context_store(): self.assertEqual(self.fixture.backend.searches, 1) +@unittest.skipUnless(os.name == "posix", "private storage needs POSIX") +class GuidedRepositoryDeliveryTests(unittest.TestCase): + """Published repository evidence must match the actual consuming checkout (issue #982). + + Unlike ``GuidedContextTests``, the connection here is a real local + repository graph over a real throwaway Git checkout, so a mismatch + between the checkout doing the work and the bound PR revision is an + actual divergence between two resolvable commits, not a mocked value. + """ + + SESSION_ID = "a" * 32 + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = graph_fixtures.make_repository(self.root) + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + self.store = ContextStore(self.private, vault=MemoryVault()) + self.associations = context_session.association_store(self.private) + graph_connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": GRAPH_RECIPIENTS, + }) + selected = context_session.create( + self.associations, + { + "id": self.SESSION_ID, "repo": "owner/repo", "host": "codex", + "orchestrator": "codex", "participants": [{"id": "claude"}, {"id": "codex"}], + }, + work_item="WORK-1", + policy=GRAPH_POLICY, + ) + report, code = context_prepare.prepare( + self.associations, selected, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + self.record = context_session.read(self.associations, self.SESSION_ID) + self.head = self.manifest.commit + self.comments: list[dict] = [] + + def _pull(self, *_args, **_kwargs): + return {"head": {"sha": self.head}} + + def _comments(self, *_args, **_kwargs): + return list(self.comments) + + def _github(self, _method, _path, **_kwargs): + return {} + + def _post(self, _repo, _pr, body, **_kwargs): + self.comments.append( + {"id": len(self.comments) + 1, "user": {"login": "controller"}, "body": body} + ) + return {"id": len(self.comments)} + + def patches(self): + return ( + mock.patch.object( + context_guided, "_github_access", return_value=("token", ("controller",)), + ), + mock.patch.object(context_guided, "fetch_pull_request", side_effect=self._pull), + mock.patch.object(context_guided, "fetch_issue_comments", side_effect=self._comments), + mock.patch.object(context_guided, "_gh_request", side_effect=self._github), + mock.patch.object(context_guided, "post_pr_comment", side_effect=self._post), + ) + + def attach(self, pr=1): + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + ): + current = context_session.read(self.associations, self.SESSION_ID) + return context_guided.attach_session( + self.associations, self.store, current, repo_path=self.repository, + pr=pr, backend=None, + ) + + def _advance_the_checkout(self): + """Commit new work so the checkout's HEAD moves off the graph's commit.""" + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + graph_fixtures.git(self.repository, "add", ".") + graph_fixtures.git(self.repository, "commit", "-q", "-m", "second") + + def _advance_without_disturbing_citations(self): + """Move HEAD to a new commit without touching any file the fixture's + synthetic graph cites, so a graph rebuilt at the new commit still + validates the same citations.""" + (self.repository / "NOTES.md").write_text("advance\n", encoding="utf-8") + graph_fixtures.git(self.repository, "add", ".") + graph_fixtures.git(self.repository, "commit", "-q", "-m", "advance") + + def test_repository_delivery_matches_the_actual_consuming_checkout(self): + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + published = context_session.read(self.associations, self.SESSION_ID) + with self.patches()[0], self.patches()[1], self.patches()[2]: + evidence = context_guided.deliver_session( + self.associations, self.store, published, repo_path=self.repository, backend=None, + ) + self.assertIn("Private evidence", evidence) + self.assertIn("example_pkg/config.py#L12", evidence) + self.assertEqual( + context_session.read(self.associations, self.SESSION_ID)["context_state"], "ready", + ) + + def test_published_delivery_refuses_when_the_actual_checkout_has_moved(self): + self.attach() + published = context_session.read(self.associations, self.SESSION_ID) + # The remote PR head the stub reports is untouched; only the actual + # consuming checkout moves, which is exactly the divergence issue + # #982 describes. + self._advance_the_checkout() + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, published, + repo_path=self.repository, backend=None, + ) + after = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(after["context_state"], "unavailable") + self.assertEqual(after["stage"], "attached") + + def test_published_repository_delivery_from_a_non_git_directory_fails_closed(self): + self.attach() + published = context_session.read(self.associations, self.SESSION_ID) + outside = self.root / "not-a-checkout" + outside.mkdir() + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, published, repo_path=outside, backend=None, + ) + self.assertEqual( + context_session.read(self.associations, self.SESSION_ID)["context_state"], + "unavailable", + ) + + def test_unpublished_repository_delivery_still_binds_the_consuming_checkout(self): + evidence = context_guided.deliver_session( + self.associations, self.store, self.record, repo_path=self.repository, backend=None, + ) + self.assertIn("Private evidence", evidence) + self._advance_the_checkout() + moved = context_session.read(self.associations, self.SESSION_ID) + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, moved, repo_path=self.repository, backend=None, + ) + + def test_fresh_attachment_refuses_a_moved_checkout_before_any_publication(self): + """A fresh reservation must bind the actual consumer, not merely the + remote PR head (codex:65a17212478a56416b1c).""" + self._advance_the_checkout() + with self.assertRaises(ContextError): + self.attach() + self.assertEqual(self.comments, []) + pending = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(pending["attachment_state"], "none") + self.assertNotEqual(pending["context_state"], "ready") + + def test_fresh_attachment_recovers_after_checkout_and_pr_advance(self): + """A fresh reservation that fails before publication must roll back + to a refreshable prepared/none state -- with zero public writes and + no leaked usable binding -- rather than stranding a permanently + pending intent that every retry reauthorizes and that + ``prepare --refresh`` alone could never clear + (codex:1a4bc7b34687da726908).""" + self._advance_without_disturbing_citations() + self.head = lifecycle.resolve_revision(self.repository)[0] + fixed = uuid.UUID(hex="1" * 32) + with mock.patch.object(context_guided.uuid, "uuid4", return_value=fixed): + with self.assertRaises(ContextError): + self.attach() + self.assertEqual(self.comments, []) + with self.assertRaises(ContextError): + read_binding(self.store, fixed.hex) + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + self.assertEqual( + (recovered["pr"], recovered["head"], recovered["revision"]), (None, None, None), + ) + self.assertEqual(recovered["stage"], "prepared") + self.assertNotEqual(recovered["context_state"], "ready") + + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + report, code = context_prepare.prepare( + self.associations, recovered, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + refresh=True, + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + attached = context_session.read(self.associations, self.SESSION_ID) + self.assertIsNotNone(attached["revision"]) + self.assertNotEqual(attached["revision"], fixed.hex) + + def test_same_head_failures_recover_without_stranding_refresh(self): + """A same-head graph generation replacement, or an unresolvable + consuming revision, must roll back exactly like a moved checkout -- + never stranding the session on a permanently pending intent + (codex:1a4bc7b34687da726908).""" + # The graph is rebuilt for the same commit, minting a new generation + # the saved packet was never authorized against. + lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + with self.assertRaises(ContextError): + self.attach() + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + self.assertEqual( + (recovered["pr"], recovered["head"], recovered["revision"]), (None, None, None), + ) + + # The consuming checkout cannot be resolved at all. + outside = self.root / "not-a-checkout" + outside.mkdir() + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + ): + current = context_session.read(self.associations, self.SESSION_ID) + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.store, current, repo_path=outside, pr=1, backend=None, + ) + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + + # The same session can still explicitly refresh and later attach. + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + report, code = context_prepare.prepare( + self.associations, recovered, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + refresh=True, + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + + def test_pending_retry_revalidates_the_actual_consumer_before_publication(self): + """A resumed pending/uncertain attachment must revalidate the current + consumer rather than replay a stale reservation, while preserving the + saved intent and its failure diagnostics (codex:65a17212478a56416b1c). + The crash is injected at the start of ``_finish_publication`` so the + reservation has already durably transitioned to ``pending`` -- still + strictly before any GitHub write -- rather than leaving the + pre-publication ``reserving`` marker this same scenario would leave + if the crash instead landed inside ``reserve_attachment`` itself.""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "_finish_publication", side_effect=KeyboardInterrupt()), + ): + with self.assertRaises(KeyboardInterrupt): + context_guided.attach_session( + self.associations, self.store, self.record, repo_path=self.repository, + pr=1, backend=None, + ) + pending = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(pending["attachment_state"], "pending") + revision = pending["revision"] + self._advance_the_checkout() + with self.assertRaises(ContextError): + self.attach() + after = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(after["attachment_state"], "pending") + self.assertEqual(after["revision"], revision) + self.assertNotEqual(after["context_state"], "ready") + self.assertEqual(self.comments, []) + + def test_reserving_retry_recovers_before_any_reservation_ever_completes(self): + """A crash inside ``reserve_attachment`` itself -- before the saved + intent ever durably transitions to ``pending`` -- leaves the durable + pre-publication ``reserving`` marker instead. A later attach must + recognize it, safely abandon the identity (idempotently, since no + binding was ever created), and restore prepared/none rather than + replaying it as if it were a genuine pending publication attempt + (codex:1a4bc7b34687da726908).""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()), + ): + with self.assertRaises(KeyboardInterrupt): + context_guided.attach_session( + self.associations, self.store, self.record, repo_path=self.repository, + pr=1, backend=None, + ) + reserving = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(reserving["attachment_state"], "reserving") + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_context_prepare.py b/tests/test_context_prepare.py index 1c34fee4..ec3c0359 100644 --- a/tests/test_context_prepare.py +++ b/tests/test_context_prepare.py @@ -5,15 +5,17 @@ import io import json import os +import secrets import tempfile import unittest +import uuid from contextlib import contextmanager, redirect_stderr, redirect_stdout from pathlib import Path from unittest import mock -from code_mower import context_prepare, context_session, session +from code_mower import context_delivery, context_guided, context_prepare, context_session, session from code_mower.context_connections import connect -from code_mower.context_contract import ContextError, ContextRetrievalError +from code_mower.context_contract import ContextError, ContextRequest, ContextRetrievalError from code_mower.context_store import ContextStore from test_context_connections import MemoryVault from test_context_packets import RetrievalBackend @@ -149,6 +151,90 @@ def test_prepare_preserves_a_pending_or_uncertain_attachment_for_reconciliation( self.backend.revoked = False self.assertEqual(prepared["packet"], saved["packet"]) + def test_prepare_preserves_a_genuine_unpublished_reservation_and_refuses_refresh(self): + """A real ``reserving`` intent -- exactly the durable, unpublished + state a crash inside ``reserve_attachment`` itself would leave + behind, strictly before any GitHub write -- must guard prepare + exactly like ``pending``/``uncertain``: the earliest check, before + checkout revision, query fingerprint, work-order, packet store, or + provider access. Neither ordinary prepare nor a changed-query + ``--refresh`` may inspect or abandon the saved binding; only + attach's own cleanup primitive may resolve it, and only then does + prepare work again (codex:1a4bc7b34687da726908).""" + record = self.create_record() + _report, code = self.prepare(record) + self.assertEqual(code, 0) + prepared = context_session.read(self.associations, record["session_id"]) + + revision = uuid.uuid4().hex + head = secrets.token_hex(20) + reserving = context_session.update( + self.associations, + prepared["session_id"], + expected_generation=prepared["generation"], + changes={ + "pr": 42, "head": head, "revision": revision, + "attachment_state": "reserving", + }, + ) + # This organization connection's evidence versions independently of + # any Git checkout, so the reservation names no consuming revision -- + # the same "no Git checkout revision" call attach would have made. + context_delivery.reserve_attachment( + self.packet_store, + reserving["connection"], + reserving["packet"], + reserving["policy"], + ContextRequest(reserving["repo"], reserving["work_item"], reserving["host"] + ":orchestrator"), + pr=42, + head=head, + revision=revision, + consuming_revision=None, + backend=self.backend, + ) + binding = context_delivery.read_binding(self.packet_store, revision) + self.assertFalse(binding["published"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + def snapshot(): + return { + path: path.read_bytes() + for path in sorted(self.private.rglob("*")) + if path.is_file() + } + + before = snapshot() + self.backend.revoked = True + + report, code = self.prepare(reserving) + self.assertEqual( + (code, report["status"], report["stage"], report["dependent_work"]), + (0, "attachment_in_progress", "attachment_pending", "paused"), + ) + self.assertEqual(self.backend.searches, 1) + self.assertEqual(snapshot(), before) + + with self.assertRaisesRegex(ContextError, "reconcile the saved attachment"): + self.prepare(reserving, refresh=True, query="a materially different bounded query") + self.assertEqual(self.backend.searches, 1) + self.assertEqual(snapshot(), before) + + self.backend.revoked = False + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual(saved, reserving) + self.assertEqual(prepared["packet"], saved["packet"]) + + context_guided._abandon_and_clear(self.associations, self.packet_store, saved) + cleared = context_session.read(self.associations, record["session_id"]) + self.assertEqual(cleared["attachment_state"], "none") + with self.assertRaises(ContextError): + context_delivery.read_binding(self.packet_store, revision) + + resumed, code = self.prepare(cleared) + self.assertEqual((code, resumed["status"], resumed["reused"]), (0, "prepared", True)) + self.assertEqual(self.backend.searches, 1) + def test_failed_or_interrupted_search_requires_explicit_refresh(self): record = self.create_record() self.backend.fail_search = True diff --git a/tests/test_context_session.py b/tests/test_context_session.py index ad0cce1c..32569f1e 100644 --- a/tests/test_context_session.py +++ b/tests/test_context_session.py @@ -178,6 +178,116 @@ def test_resolution_rejects_conflicting_trusted_values(self): with self.assertRaisesRegex(ContextError, "conflicts with the saved session"): context_session.resolve_bound("repository", "owner/repo", "owner/other") + def test_reserving_state_is_reported_like_pending_and_rejects_malformed_combinations(self): + """The durable pre-publication ``reserving`` marker validates and is + reported exactly like ``pending`` -- attach reconciliation, no owner + action -- and malformed cross-state combinations fail closed rather + than being inferred from a missing artifact (codex:1a4bc7b34687da726908).""" + record = context_session.create(self.store, self.session, work_item="SECRET-1", policy=POLICY) + prepared = context_session.update( + self.store, self.session["id"], expected_generation=record["generation"], + changes={ + "stage": "prepared", "builder": "codex", "query_mode": "work_item", + "request_hash": "c" * 64, "packet": "b" * 32, "work_order": "work-order.md", + }, + ) + reserving = context_session.update( + self.store, self.session["id"], expected_generation=prepared["generation"], + changes={ + "pr": 7, "head": "d" * 40, "revision": "e" * 32, "attachment_state": "reserving", + }, + ) + self.assertEqual(reserving["attachment_state"], "reserving") + status = context_session.status(reserving, lease_live=True) + self.assertEqual(status["stage"], "attachment_pending") + self.assertFalse(status["owner_action"]) + self.assertIn("attach", status["next_action"]) + for private in ("SECRET-1", "example-context", "owner/repo", "e" * 32): + self.assertNotIn(private, json.dumps(status)) + + for mutation in ( + # Claims a mid-preparation stage while still holding an + # attachment identity -- a malformed cross-state record. + {"stage": "preparing", "builder": None, "query_mode": None, "request_hash": None, + "packet": None, "work_order": None}, + {"pr": None}, + {"head": None}, + {"revision": None}, + ): + with self.subTest(mutation=mutation), self.assertRaises(ContextError): + context_session.validate({**copy.deepcopy(reserving), **mutation}) + + def test_reserving_precedes_context_state_failure_while_pending_and_uncertain_do_not(self): + """A ``reserving`` intent proves publication never began, so attach + reconciliation is safe without reauthorization and is reported that + way regardless of any ``context_state`` failure recorded against a + prior generation, or of required/optional policy. ``pending`` and + ``uncertain`` keep their existing failure precedence unchanged -- + their authorization failures may still need owner action + (codex:1a4bc7b34687da726908).""" + failure_stage_for = { + "ready": None, + "unchecked": None, + "unavailable": "context_unavailable", + "authorization_failed": "authorization_failed", + "expired": "context_expired", + } + counter = 0 + + def make_record(attachment_state, context_state, required): + nonlocal counter + counter += 1 + session_id = format(counter, "032x") + record = context_session.create( + self.store, session_value(session_id), work_item="SECRET-1", + policy={**POLICY, "required": required}, + ) + prepared = context_session.update( + self.store, session_id, expected_generation=record["generation"], + changes={ + "stage": "prepared", "builder": "codex", "query_mode": "work_item", + "request_hash": "c" * 64, "packet": "b" * 32, + "work_order": "work-order.md", "context_state": context_state, + }, + ) + return context_session.update( + self.store, session_id, expected_generation=prepared["generation"], + changes={ + "pr": 7, "head": "d" * 40, "revision": "e" * 32, + "attachment_state": attachment_state, + }, + ) + + for required in (True, False): + for context_state, failure_stage in failure_stage_for.items(): + with self.subTest(required=required, context_state=context_state, attachment="reserving"): + reserving = make_record("reserving", context_state, required) + status = context_session.status(reserving, lease_live=True) + self.assertEqual(status["stage"], "attachment_pending") + self.assertEqual(status["dependent_work"], "paused") + self.assertFalse(status["owner_action"]) + self.assertIn("attach", status["next_action"]) + for private in ("SECRET-1", "example-context", "owner/repo", "e" * 32): + self.assertNotIn(private, json.dumps(status)) + + for attachment in ("pending", "uncertain"): + with self.subTest(required=required, context_state=context_state, attachment=attachment): + record = make_record(attachment, context_state, required) + status = context_session.status(record, lease_live=True) + if failure_stage is not None: + self.assertEqual(status["stage"], failure_stage) + self.assertTrue(status["owner_action"]) + self.assertEqual( + status["dependent_work"], "paused" if required else "usable", + ) + else: + self.assertEqual( + status["stage"], + "attachment_uncertain" if attachment == "uncertain" else "attachment_pending", + ) + self.assertEqual(status["dependent_work"], "paused") + self.assertEqual(status["owner_action"], attachment == "uncertain") + def test_malformed_or_cross_bound_records_fail_closed(self): record = context_session.create(self.store, self.session, work_item="ITEM-1", policy=POLICY) mutations = ( diff --git a/tests/test_devin_work_orders.py b/tests/test_devin_work_orders.py index 10562296..413d27c7 100644 --- a/tests/test_devin_work_orders.py +++ b/tests/test_devin_work_orders.py @@ -621,7 +621,10 @@ def assert_paused(self, output, slot, policy="required"): self.assertNotIn("session", output) def test_packet_context_carries_no_identity_and_policy_must_agree_with_the_order(self): - self.assertEqual(tuple(self.context.__dataclass_fields__), ("store", "name", "handle", "policy", "backend")) + self.assertEqual( + tuple(self.context.__dataclass_fields__), + ("store", "name", "handle", "policy", "backend", "revision"), + ) self.assertNotIn(CONTEXT_CANARY, repr(self.context)) self.assertNotIn(self.result["packet_handle"], repr(self.context)) for handle in ("", "not-a-handle", self.result["packet_handle"].upper(), None): @@ -641,6 +644,43 @@ def test_packet_context_carries_no_identity_and_policy_must_agree_with_the_order packet_context(self.fixture.store, "example", self.result["packet_handle"], policy, order=order, backend=self.backend) + def test_the_factory_can_supply_the_consuming_revision(self): + """A repository-kind route needs the consuming commit, so a caller can name it. + + The hosted order has no PR head yet and this process is not the checkout + doing the work, so the revision is the one thing the factory cannot + derive. It reaches ``ContextRequest.revision`` on every render, which is + what a local repository graph is authorized against; leaving it unset + stays the organization-context default. + """ + commit = "c" * 40 + bound = packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=commit) + self.assertEqual(bound.revision, commit) + self.assertNotIn(commit, repr(bound)) + # Unset is still valid, and is what organization context prepares with. + self.assertIsNone(self.context.revision) + for value in ("", "a\nb", "a" * 201, 40): + with self.subTest(revision=value), self.assertRaisesRegex(RemoteError, "invalid_request"): + packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=value) + + def test_the_consuming_revision_reaches_the_context_request(self): + """Whatever the load then decides, the request it decides on names the commit.""" + commit = "d" * 40 + bound = packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=commit) + with patch("code_mower.devin_work_orders.load_authorized", wraps=load_authorized) as load: + try: + self.service.run("dispatch", self.order, context=bound, apply=True) + except RemoteError: + pass + self.assertTrue(load.call_args_list) + request = load.call_args_list[0].args[4] + self.assertEqual(request.revision, commit) + self.assertEqual((request.repository, request.work_item), + (self.order.repository, self.order.work_item)) + def test_preview_never_retrieves_context(self): with patch("code_mower.devin_work_orders.load_authorized", wraps=load_authorized) as load: for command in ("dispatch", "clarify", "fix"):