From 5edb1906f93d5ebb1e15ada78da8d1a2443cef1b Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Sun, 13 Sep 2026 23:59:30 -0700 Subject: [PATCH 01/33] Graphify: bounded query normalization and context packets Read the pinned graph schema directly out of a published generation, answer four bounded deterministic questions, and turn the answer into a revision-bound context packet the existing delivery path already carries. The lifecycle (#913) publishes an immutable generation; context_graph (#876) scores a delivered packet. Between them there was no way to ask the graph a question and no way to turn an answer into evidence a recipient may read. - Pinned schema read strictly from the artifact member, with node/edge kinds, dangling edges, duplicate ids, inverted spans and out-of-checkout paths all refusals rather than best-effort reads. - Symbol-first, relationship-filtered traversals for impact, dependency, symbol and related-test questions, breadth-first over sorted adjacency with explicit depth and node budgets; reaching a budget reports truncation. - Citations validated against the bound commit's tracked census and blobs, not the working tree; an unconfirmable location is dropped and reported. - Packets carry source revision, graph generation, completeness, truncation, omission codes and extracted/inferred/ambiguous confidence. - Required unavailable context blocks dependent work; optional unavailable context degrades to ordinary repository tools. - One approved packet reaches Claude, Codex and Devin through the shared delivery path with no provider tools or credentials in the recipient. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 186 ++++++ docs/graphify-evaluation.md | 11 +- src/code_mower/context_graph_command.py | 79 ++- src/code_mower/context_graph_query.py | 842 ++++++++++++++++++++++++ tests/test_context_graph_query.py | 565 ++++++++++++++++ 5 files changed, 1679 insertions(+), 4 deletions(-) create mode 100644 docs/context-graph-queries.md create mode 100644 src/code_mower/context_graph_query.py create mode 100644 tests/test_context_graph_query.py diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md new file mode 100644 index 00000000..341ac13e --- /dev/null +++ b/docs/context-graph-queries.md @@ -0,0 +1,186 @@ +# Local repository graph: bounded queries and context packets + +How Code Mower asks a published local graph a question, and how the answer +becomes evidence a recipient may read. Recorded for +[issue #914](https://github.com/codemower-ai/code-mower/issues/914) under epic +[#902](https://github.com/codemower-ai/code-mower/issues/902). + +This is the third and last piece of the optional local-graph path: + +- [`context_graph_lifecycle.py`](context-graph-lifecycle.md) (#913) builds and + publishes an immutable generation bound to one commit and tree. +- `context_graph_query.py` (this document) reads the pinned schema out of that + generation, answers four bounded questions, and emits a packet. +- [`context_graph.py`](graphify-evaluation.md) (#876) scores a delivered + packet's citations and freshness. + +Nothing here installs, downloads, or runs a provider, and nothing here is on a +default path. The only subprocess is Git, reading blobs of the commit the +generation is already bound to. + +## Reading the pinned schema directly + +A generation's artifact is the provider's own state, packed reproducibly. This +adapter reads one member of it, `graph.json`, and requires it to declare +`code_mower.contextGraph.v1`: + +```json +{ + "schema": "code_mower.contextGraph.v1", + "nodes": [ + {"id": "n-config", "kind": "symbol", "name": "parse_config", + "path": "example_pkg/config.py", "start_line": 12, "end_line": 30} + ], + "edges": [ + {"source": "n-load", "target": "n-config", "kind": "calls", + "evidence": "extracted"} + ] +} +``` + +Node kinds are `file`, `symbol`, `test`. Edge kinds are `calls`, `imports`, +`defines`, `references`, `tests`. Every other shape is a refusal, not a +best-effort read: an adapter that repairs what it does not understand reports a +traversal over a graph nobody reviewed. The refusals are exhaustive on purpose — +an unknown kind, a dangling edge, a duplicate identifier, an inverted line span, +an unrecognized field, a node path that escapes the indexed checkout. + +Reading the member directly, rather than through provider query tooling, is what +makes the traversal reproducible and the bounds ours. It also means a recipient +never needs the provider: the graph is read once, here, by the operator who +built it. + +The member is read as a stream and bounded as it is read. The artifact as a +whole is already inside the lifecycle's budget and its digest was verified by +`graph_status` before this module opened it; one member of it is separately +bounded because a declared size is something this process would otherwise +allocate before looking at it. + +## The four questions + +Free-form traversal is not offered. A traversal whose shape comes from the +question text cannot be bounded or reproduced, and the adoption record's second +product constraint is that default traversals in the evaluated provider returned +700–900 nodes and truncated silently. + +| Question | Direction | Relationships | Default depth | +| --- | --- | --- | --- | +| `impact` | against the edges | `calls`, `imports`, `references`, `tests` | 2 | +| `dependency` | along the edges | `calls`, `imports`, `references` | 2 | +| `symbol` | both | all | 1 | +| `related_tests` | against the edges, answering with test nodes only | `tests`, `calls`, `references` | 2 | + +Each traversal is symbol-first: a target resolves to the symbols carrying that +name, and only a target that names no symbol at all is read as a path. Each is +breadth-first over adjacency sorted by `(kind, target, source)`, so one +generation and one question produce one answer, every time, and a budget cut +removes the furthest relationships rather than arbitrary ones. + +Reaching the node budget sets `truncated` and raises `provider_has_more`. It +never silently shortens the answer. + +## Citations are validated against the bound commit + +Not against the working tree, which is the point. The generation binds one +commit; the checkout it was built from has since been edited, rebased, or left +dirty, and a line claim confirmed against an edited file is a claim confirmed +against a revision nobody asked about. + +Every cited path must appear in that commit's tracked census — so an untracked, +ignored, or since-deleted file is never cited — and every line claim is +confirmed against the blob the census names, read through `git cat-file` and +stopped at the claimed line. Blob line counts are memoized, so a packet that +cites one file many times reads it once. + +A location that cannot be confirmed is dropped rather than downgraded: the +packet's whole claim is that its citations point at the immutable tree, and +evidence that cannot be pointed at is not weaker evidence, it is none. The +drop is reported as `provider_warning`, and a relationship left with no citation +at all is reported as `document_limit`. + +The scope rules are `context_graph`'s, applied twice: at parse time, so a node +that could never be cited is not traversable either, and again at citation time. + +## What the packet carries + +An ordinary `code_mower.contextPacket.v1` repository-kind packet — the same +shape every other provider delivers, so it travels the existing delivery path +with no new contract: + +| Field | Bound to | +| --- | --- | +| `source_revision` | the generation's commit, so a consumer asking about another revision resolves `stale` at delivery | +| `source_built_at` | the manifest's build time | +| `completeness`, `truncated` | the traversal's budget *and* the generation's own completeness | +| `omissions` | `provider_has_more`, `unresolved_entities`, `provider_warning`, `document_limit`, `provider_partial` | +| `documents[].confidence` | the provider's qualification of each relationship | +| `documents[].citations` | validated file/line references into the bound commit | +| `binding` | the authorization envelope, unchanged | + +Confidence maps the provider's own qualification onto the contract's vocabulary: + +| Provider evidence | Packet confidence | Meaning | +| --- | --- | --- | +| `extracted` | `extracted` | parsed from the source | +| `inferred` | `inferred` | derived, not read directly | +| `ambiguous` | `unknown` | resolved to more than one candidate | + +`ambiguous` also raises `unresolved_entities` on the packet, so a recipient sees +the uncertainty at the packet level and not only per document. A target name +that matches more than one definition does the same. + +Document text is metadata about relationships — names, paths, relationship +kinds, hop counts — and never indexed content. Everything in it is already in +the citations beside it, so the prose adds no claim a recipient cannot check. + +## Required blocks, optional degrades + +Usability is settled before anything is read, against `graph_status`: a graph +that is absent, stale, partial, corrupt or oversized never reaches a traversal. +Neither outcome raises, because "no graph" is a normal state of an opt-in +feature. + +| Policy | Graph unusable | `dependent_work` | Exit code | +| --- | --- | --- | --- | +| `required: true` | `required_unavailable` | `paused` | 1 | +| `required: false` | `optional_unavailable` | `usable` | 0 | + +The words match `context_prepare`, so a caller branches on one vocabulary. +Optional unavailable context is not a degraded answer — there is no packet at +all, and the next action is to carry on with ordinary repository tools. + +## One packet, three recipients + +Claude, Codex and Devin receive the same approved bytes through the same +delivery path. The rendered evidence is identical for each: it names no +connection, no credential, no provider tool, and no local path, and a recipient +needs no graph, no provider install, and no Graphify credentials of its own. +Authorization is unchanged — a recipient the envelope does not name is still +refused at load. + +## Command + +``` +code-mower context-graph query --question impact --target parse_config \ + --authorization AUTH.json [--packet-out PACKET.json] \ + [--revision REV] [--depth N] [--node-budget N] [--json] +``` + +`--authorization` names a JSON file carrying the connection envelope, the +policy, the repository and the work item. It is read from a file the operator +names rather than discovered: this command mints evidence for named recipients, +and which recipients those are is an authorization decision that belongs to the +connection, not to a query. + +Standard output is metadata only — counts, states, the bound revision and +generation, and the omission codes. The evidence itself goes to the private file +named by `--packet-out`, created `0600`, or nowhere at all. + +## Boundary + +This change adds no dependency, no background service, and no mandatory +indexing step. It does not wire the graph into `code-mower context fetch` or any +default guided-context selection: a graph packet is produced by an explicit +command, and the operator attaches it through the ordinary path. Hosted +Graphify, semantic or model-based extraction, clustering, watchers, and provider +API keys remain out of scope and separate decisions. diff --git a/docs/graphify-evaluation.md b/docs/graphify-evaluation.md index 22af78fc..f046edcd 100644 --- a/docs/graphify-evaluation.md +++ b/docs/graphify-evaluation.md @@ -174,8 +174,15 @@ lifecycle in outside every checkout, a manifest that binds full commit and tree with build time, opt-in acquisition with no default dependency, and a `partial` completeness state that refuses to read a fast incremental repeat as a complete -graph. Conditions 4 and 6 belong to the retrieval adapter, which does not exist -yet. +graph. + +Conditions 4 and 6 are implemented by the query and packet adapter in +[Local repository graph: bounded queries and context packets](context-graph-queries.md) +(issue #914): symbol-first, relationship-filtered traversals with explicit +depth and node budgets, truncation reported as `truncated` plus a +`provider_has_more` omission rather than hidden, and a packet whose citations +are validated against the bound commit's tracked tree before any of it reaches +a recipient. ## Boundary diff --git a/src/code_mower/context_graph_command.py b/src/code_mower/context_graph_command.py index a78108f7..771047ad 100644 --- a/src/code_mower/context_graph_command.py +++ b/src/code_mower/context_graph_command.py @@ -16,15 +16,23 @@ import argparse import json +import os import sys from pathlib import Path from . import context_graph_lifecycle as lifecycle +from . import context_graph_query as query from .context_contract import ContextError from .context_store import strict_json MAX_PIN_BYTES = 8192 +#: The authorization envelope and policy a packet binds to. Read from a file +#: the operator names, never discovered: this command mints evidence for named +#: recipients, and which recipients those are is an authorization decision that +#: belongs to the connection, not to a query. +MAX_AUTHORIZATION_BYTES = 65_536 + def _load_pin(path: Path | None) -> lifecycle.GraphifyPin | None: if path is None: @@ -48,6 +56,38 @@ def _require_pin(path: Path | None) -> lifecycle.GraphifyPin: return pin +def _load_authorization(path: Path) -> dict: + """The connection envelope and policy one packet will bind to. + + Bounded at the stream for the same reason the pin is: a bound checked on + bytes already in memory is not a bound on what the file can cost to read. + """ + try: + with path.open("rb") as stream: + raw = stream.read(MAX_AUTHORIZATION_BYTES + 1) + except OSError: + raise ContextError("context authorization file is unreadable") from None + if len(raw) > MAX_AUTHORIZATION_BYTES: + raise ContextError("context authorization file exceeds its bound") + payload = strict_json(raw) + if not isinstance(payload, dict) or set(payload) != {"connection", "policy", "repository", "work_item"}: + raise ContextError("context authorization must carry a connection, policy, repository and work item") + return payload + + +def _write_packet(destination: Path, packet: dict) -> None: + """Write one packet where only this operator can read it. + + ``0o600`` at creation rather than after: the delivery path refuses a packet + whose mode ever allowed anyone else, and a chmod after the fact is a window + in which it did. + """ + body = json.dumps(packet, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":")) + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(body) + + def _emit(payload: dict, *, as_json: bool, text: str) -> None: if as_json: print(json.dumps(payload, indent=2, sort_keys=True)) @@ -66,12 +106,13 @@ def main(argv=None) -> int: status = sub.add_parser("status", help="Report whether the published generation may be used") remove = sub.add_parser("remove", help="Delete this checkout's private local graph state") doctor = sub.add_parser("doctor", help="Check the local graph posture without building anything") + ask = sub.add_parser("query", help="Answer one bounded question and emit a revision-bound packet") - for command in (build, refresh, status, remove, doctor): + for command in (build, refresh, status, remove, doctor, ask): command.add_argument("--repo-path", type=Path, default=Path.cwd(), help="Checkout to bind") command.add_argument("--state-dir", type=Path, help="Private state root; defaults to the context store") command.add_argument("--json", action="store_true", help="Emit a machine-readable summary") - for command in (build, refresh, status, doctor): + for command in (build, refresh, status, doctor, ask): command.add_argument("--revision", default="HEAD", help="Revision to bind, for example a commit or tag") for command in (build, refresh, doctor): command.add_argument("--pin-file", type=Path, help="JSON file naming one exact provider release") @@ -82,6 +123,15 @@ def main(argv=None) -> int: status.add_argument("--allow-partial", action="store_true", help="Treat a provider-declared partial build as usable") remove.add_argument("--show-local-paths", action="store_true", help="Include the private state path in output") + ask.add_argument("--question", required=True, choices=query.QUESTIONS, help="Which bounded question to ask") + ask.add_argument("--target", required=True, help="Symbol name, or a repository-relative path") + ask.add_argument("--authorization", type=Path, required=True, + help="JSON file with the connection envelope, policy, repository and work item") + ask.add_argument("--packet-out", type=Path, + help="Write the private packet to this file, created 0600") + ask.add_argument("--depth", type=int, help="Traversal depth; defaults to the question's own ceiling") + ask.add_argument("--node-budget", type=int, default=query.DEFAULT_NODE_BUDGET, + help="Most relationships one answer may carry before it reports truncation") args = parser.parse_args(argv) try: @@ -137,6 +187,31 @@ def main(argv=None) -> int: # A non-current graph is a normal, reportable condition, not a # command failure; exit 1 so a script can branch on usability. return 0 if report.usable else 1 + if args.command == "query": + authorization = _load_authorization(args.authorization) + outcome = query.graph_context( + args.repo_path, + question=args.question, + target=args.target, + envelope=authorization["connection"], + policy=authorization["policy"], + context_repository=authorization["repository"], + work_item=authorization["work_item"], + root=args.state_dir, + revision=args.revision, + depth=args.depth, + node_budget=args.node_budget, + ) + if outcome.packet is not None and args.packet_out is not None: + _write_packet(args.packet_out, outcome.packet) + # Metadata only, here as everywhere in this command: the summary + # carries counts, states and the binding, and the evidence itself + # goes to the private file the operator named or nowhere at all. + lines = [f"Local graph query: {outcome.status}"] + lines.extend(f" {key}: {value}" for key, value in sorted(outcome.summary.items()) + if key not in ("schema", "status")) + _emit(outcome.summary, as_json=args.json, text="\n".join(lines) + "\n") + return outcome.exit_code if args.command == "remove": state = lifecycle.GraphStateRoot(args.repo_path, root=args.state_dir) path = str(state.path) if args.show_local_paths else None diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py new file mode 100644 index 00000000..4e901ac9 --- /dev/null +++ b/src/code_mower/context_graph_query.py @@ -0,0 +1,842 @@ +"""Bounded local-graph queries and revision-bound context packets (issue #914). + +``context_graph_lifecycle`` (#913) publishes an immutable generation: an +artifact bound to one commit and tree, with a manifest that says which provider +release produced it and whether that run finished. ``context_graph`` (#876) +scores a delivered packet's citations. Between those two there was nothing: no +way to ask the graph a question and no way to turn an answer into evidence a +recipient may read. + +This module is that middle. It reads the pinned graph schema out of the +published artifact directly rather than through provider query tooling, runs +deterministic bounded traversals for the four questions a code graph answers +better than ordinary repository tools, validates every citation against the +immutable tracked tree of the bound commit, and emits an ordinary +``code_mower.contextPacket.v1`` repository-kind packet. The packet is the whole +interface to a recipient: Claude, Codex and Devin read the same approved bytes +through the existing delivery path, with no graph, no provider install, and no +credentials of their own. + +Two properties the adoption record (``docs/graphify-evaluation.md``) asks for +are load-bearing here and are therefore not configurable: + +* **Queries are symbol-first, relationship-filtered and budgeted.** Default + traversals in the evaluated provider returned 700-900 nodes and truncated + silently. Every traversal here starts from named seeds, follows one filtered + relationship set, and stops at an explicit budget that is reported as + truncation rather than presented as a complete answer. +* **Stale or unknown graph state is never answered from.** A required context + request blocks; an optional one degrades to ordinary repository tools. There + is no third outcome where a consumer is handed an older graph that looks + fresh. + +Nothing here installs, downloads, or executes a provider. The only subprocess +is Git, reading blobs of the commit the generation is already bound to. +""" + +from __future__ import annotations + +import json +import subprocess +import tarfile +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from . import context_graph_lifecycle as lifecycle +from .context_contract import ( + CAPABILITY_VERSION, + PACKET_SCHEMA, + ContextError, + _text, + _timestamp, + normalize_policy, + validate_connection, +) +from .context_graph import MAX_GRAPH_CITATIONS, parse_graph_citation + +#: The graph document Code Mower reads, by name and by schema. Pinned on both: +#: the artifact is whatever the pinned provider release wrote, and a member +#: that is merely *shaped* like a graph is not the schema this adapter was +#: reviewed against. An artifact without it is unreadable rather than +#: best-effort -- see ``read_graph``. +GRAPH_MEMBER = "graph.json" +GRAPH_SCHEMA = "code_mower.contextGraph.v1" + +QUERY_SCHEMA = "code_mower.contextGraphQuery.v1" + +#: The questions a code graph answers better than ``rg`` and an ordinary +#: reading of the tree, from the comparison set in the adoption record. Each +#: names one direction and one relationship filter; there is no free-form +#: traversal, because a traversal whose shape comes from the question text +#: cannot be bounded or reproduced. +QUESTIONS = ("impact", "dependency", "symbol", "related_tests") + +#: Node kinds and edge kinds of the pinned schema. An unrecognized kind is a +#: refusal, not a node this adapter quietly ignores: a graph that carries +#: relationships this code does not model would have its traversals silently +#: truncated by the model rather than by a budget anyone reported. +NODE_KINDS = frozenset({"file", "symbol", "test"}) +EDGE_KINDS = frozenset({"calls", "imports", "defines", "references", "tests"}) + +#: How the provider's own qualification of a relationship maps onto the packet +#: contract's confidence vocabulary. ``ambiguous`` is the important one: the +#: provider resolved the relationship to more than one candidate, which is a +#: claim a recipient must be able to see as unresolved rather than read as +#: fact. The contract has no ``ambiguous`` value, so it maps to ``unknown`` +#: and also raises the ``unresolved_entities`` omission on the packet. +EVIDENCE_CONFIDENCE = {"extracted": "extracted", "inferred": "inferred", "ambiguous": "unknown"} + +#: Relationship filters per question, and whether the traversal runs along +#: edges or against them. ``impact`` asks who is affected by a change, which is +#: the reverse of ``dependency`` over the same relationships. +_TRAVERSALS: dict[str, tuple[str, frozenset[str]]] = { + "impact": ("incoming", frozenset({"calls", "imports", "references", "tests"})), + "dependency": ("outgoing", frozenset({"calls", "imports", "references"})), + "symbol": ("both", frozenset({"defines", "calls", "imports", "references", "tests"})), + "related_tests": ("incoming", frozenset({"tests", "calls", "references"})), +} + +#: Depth ceilings per question. ``symbol`` is a neighbourhood, not a walk: +#: what defines this name and what touches it directly. +_DEFAULT_DEPTH = {"impact": 2, "dependency": 2, "symbol": 1, "related_tests": 2} +MAX_DEPTH = 4 + +#: Budgets. The node budget is what stops a traversal; the rest bound what one +#: packet may carry and are enforced before the contract's own limits so that +#: an over-budget answer is reported as truncated rather than refused at +#: delivery. +DEFAULT_NODE_BUDGET = 40 +MAX_NODE_BUDGET = 200 +MAX_DOCUMENTS = 16 +MAX_CITATIONS_PER_DOCUMENT = 10 +MAX_SEEDS = 8 + +#: Graph bounds. The artifact is already bounded to ``MAX_ARTIFACT_BYTES`` by +#: the lifecycle and its digest is verified before this module reads it, so +#: these bound what one *member* may cost this process to hold and parse. +MAX_GRAPH_BYTES = 64 * 1024 * 1024 +MAX_NODES = 200_000 +MAX_EDGES = 500_000 + +#: How much of a blob to read while confirming one line claim. A claim is +#: confirmed as soon as its last claimed line is seen, so a citation into a +#: large generated file costs the lines up to the claim. +_BLOB_CHUNK_BYTES = 256 * 1024 + + +@dataclass(frozen=True) +class GraphNode: + """One node of the pinned schema, already held to the citation scope rules.""" + + id: str + kind: str + name: str + path: str + start_line: int | None + end_line: int | None + + @property + def citation(self) -> str: + """The node's location as a citation string, with its line span if it has one.""" + if self.start_line is None: + return self.path + if self.end_line is None or self.end_line == self.start_line: + return f"{self.path}#L{self.start_line}" + return f"{self.path}#L{self.start_line}-L{self.end_line}" + + +@dataclass(frozen=True) +class GraphEdge: + """One relationship, with the provider's own qualification of it.""" + + source: str + target: str + kind: str + evidence: str + + +@dataclass(frozen=True) +class CodeGraph: + """A parsed, bounded, deterministically ordered graph of one generation.""" + + generation: str + commit: str + nodes: Mapping[str, GraphNode] + edges: tuple[GraphEdge, ...] + outgoing: Mapping[str, tuple[GraphEdge, ...]] + incoming: Mapping[str, tuple[GraphEdge, ...]] + + def seeds(self, target: str) -> tuple[GraphNode, ...]: + """Nodes a query target names, by symbol name first and then by path. + + Symbol-first, as the adoption record requires: a bare name resolves to + the symbols that carry it, and only a target that names no symbol at + all is read as a path. Ordered by id so two runs against one generation + seed identically. + """ + name = _text(target, maximum=512) + matches = [node for node in self.nodes.values() if node.name == name] + if not matches: + matches = [node for node in self.nodes.values() if node.path == name] + return tuple(sorted(matches, key=lambda node: node.id))[:MAX_SEEDS] + + +def _member(value: Any, keys: set[str], *, what: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping) or set(value) != keys: + raise ContextError(f"local graph {what} fields are missing or unrecognized") + return value + + +def _line(value: Any) -> int | None: + if value is None: + return None + if type(value) is not int or not 1 <= value <= 10_000_000: + raise ContextError("local graph line number is out of range") + return value + + +def _node(value: Any) -> GraphNode: + record = _member(value, {"id", "kind", "name", "path", "start_line", "end_line"}, what="node") + kind = record["kind"] + if kind not in NODE_KINDS: + raise ContextError("unsupported local graph node kind") + start = _line(record["start_line"]) + end = _line(record["end_line"]) + if start is None and end is not None: + raise ContextError("local graph line span must start before it ends") + if start is not None and end is not None and end < start: + raise ContextError("local graph line span must start before it ends") + # Held to the citation rules here, at parse time, rather than when a packet + # is written: a node that could never be cited inside the indexed checkout + # must not be traversable either, or an out-of-scope path reaches a + # recipient as a relationship whose citation was quietly dropped. + node = GraphNode( + id=_text(record["id"], maximum=512), + kind=kind, + name=_text(record["name"], maximum=512), + path=_text(record["path"], maximum=1024), + start_line=start, + end_line=end, + ) + parse_graph_citation(node.citation) + return node + + +def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge: + record = _member(value, {"source", "target", "kind", "evidence"}, what="edge") + if record["kind"] not in EDGE_KINDS: + raise ContextError("unsupported local graph edge kind") + if record["evidence"] not in EVIDENCE_CONFIDENCE: + raise ContextError("unsupported local graph edge evidence") + source = _text(record["source"], maximum=512) + target = _text(record["target"], maximum=512) + if source not in nodes or target not in nodes: + raise ContextError("local graph edge names a node the graph does not carry") + return GraphEdge(source=source, target=target, kind=record["kind"], evidence=record["evidence"]) + + +def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdge, ...]]: + """Adjacency in one fixed order, so a traversal cannot depend on input order.""" + buckets: dict[str, list[GraphEdge]] = {} + for edge in edges: + buckets.setdefault(getattr(edge, by), []).append(edge) + return { + key: tuple(sorted(group, key=lambda edge: (edge.kind, edge.target, edge.source))) + for key, group in buckets.items() + } + + +def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> CodeGraph: + """Validate the pinned graph schema. Every unreadable shape is a refusal. + + Read strictly for the same reason the manifest is: this document is + provider output, and an adapter that repairs what it does not understand + reports a traversal over a graph nobody reviewed. + """ + document = _member(payload, {"schema", "nodes", "edges"}, what="document") + if document["schema"] != GRAPH_SCHEMA: + raise ContextError("unsupported local graph schema") + raw_nodes = document["nodes"] + raw_edges = document["edges"] + if not isinstance(raw_nodes, list) or len(raw_nodes) > MAX_NODES: + raise ContextError("local graph node count exceeds its budget") + if not isinstance(raw_edges, list) or len(raw_edges) > MAX_EDGES: + raise ContextError("local graph edge count exceeds its budget") + nodes: dict[str, GraphNode] = {} + for value in raw_nodes: + node = _node(value) + if node.id in nodes: + raise ContextError("local graph node identifiers must be unique") + nodes[node.id] = node + edges = tuple(sorted( + (_edge(value, nodes) for value in raw_edges), + key=lambda edge: (edge.kind, edge.source, edge.target), + )) + return CodeGraph( + generation=generation, + commit=commit, + nodes=nodes, + edges=edges, + outgoing=_grouped(edges, by="source"), + incoming=_grouped(edges, by="target"), + ) + + +def read_graph(state: lifecycle.GraphStateRoot, status: lifecycle.GenerationStatus) -> CodeGraph: + """Read the pinned schema out of a generation whose digest already matched. + + ``status`` must be a usable verdict from ``graph_status``: that is what + bound the artifact to a commit and verified its digest, and re-deriving + either here would be a second answer to a question already settled. + + The member is read from the archive as a stream and bounded as it is read. + The whole artifact is already inside the lifecycle's budget, but one member + of it is not: a compressed or sparse entry can declare a size this process + would otherwise allocate before looking at it. + """ + if not status.usable or status.manifest is None or status.generation is None: + raise ContextError("local graph is not usable for queries") + path = state.artifact_path(status.generation) + try: + with tarfile.open(path, mode="r:") as archive: + try: + info = archive.getmember(GRAPH_MEMBER) + except KeyError: + raise ContextError("local graph generation carries no pinned graph document") from None + if not info.isfile() or info.size > MAX_GRAPH_BYTES: + raise ContextError("local graph document is missing or exceeds its budget") + stream = archive.extractfile(info) + if stream is None: + raise ContextError("local graph document is unreadable") + raw = stream.read(MAX_GRAPH_BYTES + 1) + except (OSError, tarfile.TarError): + raise ContextError("local graph artifact is unreadable") from None + if len(raw) > MAX_GRAPH_BYTES: + raise ContextError("local graph document exceeds its budget") + try: + payload = json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + raise ContextError("local graph document is not supported JSON") from None + return load_graph(payload, generation=status.generation, commit=status.manifest.commit) + + +@dataclass(frozen=True) +class Relation: + """One traversal result: a reached node, how it was reached, and from where.""" + + node: GraphNode + via: GraphEdge + origin: GraphNode + depth: int + + +@dataclass(frozen=True) +class QueryResult: + """A bounded traversal, with everything a consumer needs to distrust it.""" + + question: str + target: str + generation: str + commit: str + seeds: tuple[GraphNode, ...] + relations: tuple[Relation, ...] + truncated: bool + ambiguous: bool + omissions: tuple[str, ...] + + @property + def resolved(self) -> bool: + return bool(self.seeds) + + def shareable_summary(self) -> dict[str, Any]: + """Metadata only: counts, states and the binding, never indexed content.""" + return { + "schema": QUERY_SCHEMA, + "question": self.question, + "generation": self.generation, + "source_revision": self.commit, + "seeds": len(self.seeds), + "relations": len(self.relations), + "truncated": self.truncated, + "ambiguous": self.ambiguous, + "omissions": list(self.omissions), + } + + +def _neighbours(graph: CodeGraph, node_id: str, direction: str, kinds: frozenset[str]) -> list[tuple[GraphEdge, str]]: + """Filtered adjacency in a fixed order: the only place direction is read.""" + found: list[tuple[GraphEdge, str]] = [] + if direction in ("outgoing", "both"): + found.extend((edge, edge.target) for edge in graph.outgoing.get(node_id, ()) if edge.kind in kinds) + if direction in ("incoming", "both"): + found.extend((edge, edge.source) for edge in graph.incoming.get(node_id, ()) if edge.kind in kinds) + return found + + +def run_query( + graph: CodeGraph, + *, + question: str, + target: str, + depth: int | None = None, + node_budget: int = DEFAULT_NODE_BUDGET, +) -> QueryResult: + """One deterministic, bounded, relationship-filtered traversal. + + Breadth-first from the seeds in a fixed order, so the same generation and + the same question produce the same answer every time and a budget cut + removes the *furthest* relationships rather than arbitrary ones. Reaching + the budget sets ``truncated``; it never silently shortens the answer. + """ + if question not in QUESTIONS: + raise ContextError("unsupported local graph question") + if type(node_budget) is not int or not 1 <= node_budget <= MAX_NODE_BUDGET: + raise ContextError("local graph node budget is out of range") + limit = _DEFAULT_DEPTH[question] if depth is None else depth + if type(limit) is not int or not 1 <= limit <= MAX_DEPTH: + raise ContextError("local graph traversal depth is out of range") + direction, kinds = _TRAVERSALS[question] + seeds = graph.seeds(target) + omissions: list[str] = [] + if not seeds: + return QueryResult( + question=question, target=target, generation=graph.generation, commit=graph.commit, + seeds=(), relations=(), truncated=False, ambiguous=False, + omissions=("unresolved_entities",), + ) + # More than one definition carries the target's name, so every relationship + # below is reported from a seed set the provider could not disambiguate. + ambiguous = len(seeds) > 1 + seen = {node.id for node in seeds} + relations: list[Relation] = [] + truncated = False + frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] + while frontier: + node, origin, level = frontier.pop(0) + if level >= limit: + continue + for edge, other_id in _neighbours(graph, node.id, direction, kinds): + if other_id in seen: + continue + if len(relations) >= node_budget: + truncated = True + break + seen.add(other_id) + reached = graph.nodes[other_id] + relations.append(Relation(node=reached, via=edge, origin=origin, depth=level + 1)) + frontier.append((reached, origin, level + 1)) + if truncated: + break + if question == "related_tests": + # Relationship-filtered is not the same as answer-filtered: the walk + # reaches callers so that a test two hops away is found, but only the + # tests are the answer. + relations = [item for item in relations if item.node.kind == "test"] + if truncated: + omissions.append("provider_has_more") + if ambiguous or any(item.via.evidence == "ambiguous" for item in relations): + omissions.append("unresolved_entities") + return QueryResult( + question=question, target=target, generation=graph.generation, commit=graph.commit, + seeds=seeds, relations=tuple(relations), truncated=truncated, ambiguous=ambiguous, + omissions=tuple(dict.fromkeys(omissions)), + ) + + +class CitationValidator: + """Confirm citations against the immutable tracked tree of the bound commit. + + Not against the working tree, which is the point. The generation binds one + commit; the checkout it was built from has since been edited, rebased, or + left dirty, and a line claim confirmed against an edited file is a claim + confirmed against a revision nobody asked about. Every path is checked + against that commit's census -- so an untracked, ignored, or since-deleted + file is never cited -- and every line claim is confirmed against the blob + the census names. + + Blob line counts are memoized per blob, so a packet that cites one file + many times reads it once, and each read stops at the claimed line. + """ + + def __init__(self, repository: Path, census: lifecycle.TrackedCensus): + self._repository = repository + self._blobs = {entry.path: entry for entry in census.entries} + self._counts: dict[str, int | None] = {} + + def tracked(self, path: str) -> bool: + return path in self._blobs + + def validate(self, citation: str) -> bool: + """True when the path is tracked at the bound commit and the lines exist.""" + try: + parsed = parse_graph_citation(citation) + except ContextError: + return False + entry = self._blobs.get(parsed.path) + if entry is None: + return False + if parsed.start_line is None: + return True + claimed = parsed.end_line or parsed.start_line + return claimed <= self._lines(entry) + + def _lines(self, entry: lifecycle.TrackedEntry) -> int: + cached = self._counts.get(entry.blob) + if cached is not None: + return cached + counted = self._count(entry) + self._counts[entry.blob] = counted + return counted + + def _count(self, entry: lifecycle.TrackedEntry) -> int: + """Lines in one blob of the bound commit, read as a stream. + + A blob is read through ``git cat-file``, never off the working tree: + the file at that path today may be a different file, or not exist. An + unreadable blob counts as zero lines, which makes every claim on it + unresolved rather than silently accepted. + """ + if entry.size == 0: + return 0 + try: + process = subprocess.Popen( + ["git", "-C", str(self._repository), "--no-optional-locks", + "cat-file", "blob", entry.blob], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=lifecycle.git_environment(), + ) + except (OSError, ValueError): + return 0 + lines = 0 + trailing = False + try: + assert process.stdout is not None + while True: + chunk = process.stdout.read(_BLOB_CHUNK_BYTES) + if not chunk: + break + lines += chunk.count(b"\n") + trailing = not chunk.endswith(b"\n") + except OSError: + return 0 + finally: + if process.stdout is not None: + try: + process.stdout.close() + except OSError: + pass + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: # pragma: no cover - git does not hang on a closed pipe + process.kill() + process.wait() + if process.returncode != 0: + return 0 + # A file whose last line has no terminator still has that line. + return lines + 1 if trailing else lines + + +def _relation_text(question: str, item: Relation) -> str: + """One sentence of metadata about a relationship. Never indexed content. + + Names, paths and relationship kinds only: everything here is already in the + citations beside it, so the prose adds no claim a recipient cannot check. + """ + verb = { + "calls": "calls", "imports": "imports", "defines": "defines", + "references": "references", "tests": "tests", + }[item.via.kind] + if item.via.source == item.node.id: + subject, object_ = item.node.name, item.origin.name + else: + subject, object_ = item.origin.name, item.node.name + return ( + f"{question}: {subject} {verb} {object_} " + f"({item.via.evidence}, hop {item.depth}, {item.node.kind} at {item.node.path})" + ) + + +@dataclass(frozen=True) +class PacketDraft: + """A packet and the metadata-only account of what it left out.""" + + packet: dict[str, Any] = field(repr=False) + summary: dict[str, Any] + + +def _documents( + result: QueryResult, validator: CitationValidator +) -> tuple[list[dict[str, Any]], list[str]]: + """One document per relationship, with only citations that actually resolve. + + A relationship whose own location cannot be confirmed against the bound + commit is dropped rather than downgraded: the packet's whole claim is that + its citations point at the immutable tree, and evidence that cannot be + pointed at is not weaker evidence, it is none. + """ + omissions: list[str] = [] + documents: list[dict[str, Any]] = [] + dropped = False + unvalidated = False + citations_used = 0 + for item in result.relations: + if len(documents) >= MAX_DOCUMENTS: + dropped = True + break + # ``dict.fromkeys`` rather than a set: a self-referential relationship + # cites one location once, in a fixed order. + candidates = list(dict.fromkeys((item.node.citation, item.origin.citation))) + cited = [candidate for candidate in candidates if validator.validate(candidate)] + if len(cited) != len(candidates): + # The graph claimed a location the bound commit does not carry. + # That is the provider disagreeing with the immutable tree, which + # a recipient must be told about even when the relationship keeps + # a second citation that does check out. + unvalidated = True + cited = cited[:MAX_CITATIONS_PER_DOCUMENT] + if not cited: + dropped = True + continue + if citations_used + len(cited) > MAX_GRAPH_CITATIONS: + dropped = True + break + citations_used += len(cited) + documents.append({ + "text": _relation_text(result.question, item), + "confidence": EVIDENCE_CONFIDENCE[item.via.evidence], + "source_kind": "local_repository_graph", + "citations": [ + {"source": source, "title": f"{item.node.kind} {item.node.name}"} + for source in cited + ], + }) + if unvalidated: + omissions.append("provider_warning") + if dropped: + omissions.append("document_limit") + return documents, omissions + + +def build_packet( + result: QueryResult, + *, + validator: CitationValidator, + envelope: Mapping[str, Any], + policy: Mapping[str, Any], + repository: str, + work_item: str, + built_at: str, + completeness: str, + now: datetime | None = None, +) -> PacketDraft: + """Turn one traversal into a revision-bound packet a recipient may read. + + The packet binds the graph's commit as ``source_revision`` and the + generation as its provenance, so a consumer that asked about a different + revision resolves ``stale`` at delivery without decoding the payload. The + binding is the authorization envelope's, unchanged: this module decides + what the evidence is, never who may read it. + """ + current = now or datetime.now(timezone.utc) + if current.tzinfo is None: + raise ContextError("context packet time must include a timezone") + limits = normalize_policy(policy) + if limits is None: + raise ContextError("context policy is not configured") + connection = validate_connection(envelope, now=current) + if connection["kind"] != "repository": + raise ContextError("local graph evidence requires a repository-kind connection") + if completeness not in (lifecycle.COMPLETE, lifecycle.PARTIAL): + raise ContextError("unsupported local graph completeness") + if not result.resolved: + raise ContextError("local graph query resolved no symbol or path to cite") + documents, dropped = _documents(result, validator) + if not documents: + raise ContextError("local graph query produced no citable evidence") + omissions = list(dict.fromkeys([ + *result.omissions, + *dropped, + # The generation's own admission, carried through rather than + # recomputed: a partial build's answer is partial however complete this + # traversal was. + *(["provider_partial"] if completeness == lifecycle.PARTIAL else []), + ])) + truncated = result.truncated or "document_limit" in omissions + packet_completeness = "partial" if truncated or completeness == lifecycle.PARTIAL else "complete" + expiry = min( + _timestamp(connection["expires_at"]), + current + timedelta(seconds=limits["max_age_seconds"]), + ) + packet = { + "schema": PACKET_SCHEMA, + "capability_version": CAPABILITY_VERSION, + "provider": connection["provider"], + "kind": "repository", + "retrieved_at": current.isoformat(), + "source_revision": result.commit, + "source_built_at": built_at, + "completeness": packet_completeness, + "truncated": truncated, + "omissions": omissions, + "documents": documents, + "binding": { + **{key: connection[key] for key in ("connection", "generation", "identity", "recipients")}, + "repository": repository, + "work_item": work_item, + "policy_version": limits["policy_version"], + "expires_at": expiry.isoformat(), + }, + } + summary = { + **result.shareable_summary(), + "graph_generation": result.generation, + "documents": len(documents), + "completeness": packet_completeness, + "truncated": truncated, + "omissions": omissions, + "recipients": list(connection["recipients"]), + } + return PacketDraft(packet=packet, summary=summary) + + +#: What a caller does next when the graph cannot answer. ``required_unavailable`` +#: is the blocking outcome and ``optional_unavailable`` the degrading one; the +#: words match ``context_prepare`` so a caller branches on one vocabulary. +AVAILABLE = "available" +REQUIRED_UNAVAILABLE = "required_unavailable" +OPTIONAL_UNAVAILABLE = "optional_unavailable" + + +@dataclass(frozen=True) +class GraphContext: + """The answer, or the reason there is none and what that means for the work.""" + + status: str + summary: dict[str, Any] + packet: dict[str, Any] | None = field(default=None, repr=False) + + @property + def dependent_work(self) -> str: + return "paused" if self.status == REQUIRED_UNAVAILABLE else "usable" + + @property + def exit_code(self) -> int: + return 1 if self.status == REQUIRED_UNAVAILABLE else 0 + + +def _unavailable(required: bool, reason: str, detail: Mapping[str, Any] | None = None) -> GraphContext: + return GraphContext( + status=REQUIRED_UNAVAILABLE if required else OPTIONAL_UNAVAILABLE, + summary={ + "schema": QUERY_SCHEMA, + "status": REQUIRED_UNAVAILABLE if required else OPTIONAL_UNAVAILABLE, + "reason": reason, + "dependent_work": "paused" if required else "usable", + "next_action": ( + "Refresh or build the local graph for this revision, then retry; dependent work is blocked." + if required else + "Continue with ordinary repository tools; optional graph context was not used." + ), + **(dict(detail) if detail else {}), + }, + ) + + +def graph_context( + repository: Path, + *, + question: str, + target: str, + envelope: Mapping[str, Any], + policy: Mapping[str, Any], + context_repository: str, + work_item: str, + root: Path | None = None, + revision: str = "HEAD", + depth: int | None = None, + node_budget: int = DEFAULT_NODE_BUDGET, + now: datetime | None = None, +) -> GraphContext: + """Answer one question from the published generation, or say why not. + + This is the whole decision in one call, and the order matters. Usability is + settled first, against ``graph_status``, which is what binds the revision + and verifies the artifact digest: a graph that is absent, stale, partial, + corrupt or oversized never reaches a traversal. Only then is the graph + read, queried, and turned into evidence. + + Required context that is unavailable blocks the dependent work. Optional + context that is unavailable returns ``optional_unavailable`` with + ``dependent_work`` usable, which is the contract's way of saying: carry on + with ordinary repository tools. Neither outcome raises, because "no graph" + is a normal state of an opt-in feature, not a failure of the caller. + """ + limits = normalize_policy(policy) + if limits is None: + raise ContextError("context policy is not configured") + required = bool(limits["required"]) + if question not in QUESTIONS: + raise ContextError("unsupported local graph question") + state = lifecycle.GraphStateRoot(repository, root=root) + status = lifecycle.graph_status(repository, root=root, revision=revision) + if not status.usable: + return _unavailable(required, status.state, {"detail": status.detail}) + manifest = status.manifest + assert manifest is not None # a usable status always carries one + try: + graph = read_graph(state, status) + census = lifecycle.read_tracked_census(state.repository, manifest.commit) + except ContextError as error: + return _unavailable(required, "unreadable", {"detail": str(error)}) + result = run_query(graph, question=question, target=target, depth=depth, node_budget=node_budget) + if not result.resolved: + return _unavailable(required, "unresolved", {"detail": "the graph carries no such symbol or path"}) + try: + draft = build_packet( + result, + validator=CitationValidator(state.repository, census), + envelope=envelope, + policy=policy, + repository=context_repository, + work_item=work_item, + built_at=manifest.built_at, + completeness=manifest.completeness, + now=now, + ) + except ContextError as error: + return _unavailable(required, "uncitable", {"detail": str(error)}) + return GraphContext( + status=AVAILABLE, + summary={"schema": QUERY_SCHEMA, "status": AVAILABLE, "dependent_work": "usable", **draft.summary}, + packet=draft.packet, + ) + + +__all__: Sequence[str] = ( + "AVAILABLE", + "CitationValidator", + "CodeGraph", + "DEFAULT_NODE_BUDGET", + "EVIDENCE_CONFIDENCE", + "GRAPH_MEMBER", + "GRAPH_SCHEMA", + "GraphContext", + "GraphEdge", + "GraphNode", + "MAX_NODE_BUDGET", + "OPTIONAL_UNAVAILABLE", + "PacketDraft", + "QUESTIONS", + "QUERY_SCHEMA", + "QueryResult", + "REQUIRED_UNAVAILABLE", + "Relation", + "build_packet", + "graph_context", + "load_graph", + "read_graph", + "run_query", +) diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py new file mode 100644 index 00000000..8526d30c --- /dev/null +++ b/tests/test_context_graph_query.py @@ -0,0 +1,565 @@ +"""Offline tests for bounded local-graph queries and packets (issue #914). + +Every test builds a real throwaway Git repository, publishes a generation +through the #913 lifecycle with an injected indexer, and asks the resulting +graph a question. No graph package is installed, imported, or required: the +graph document is written by the injected indexer, so what is proved here is +what this repository is responsible for -- how a traversal is bounded, what a +packet binds, which citations survive validation against the immutable tree, +and what happens when the graph cannot answer. + +The graph content is synthetic, as in ``tests/fixtures/local_graph_contract.json``. +A passing suite is not evidence that any provider emits this shape. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import subprocess +import tarfile +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +from code_mower import context_contract as contract +from code_mower import context_delivery +from code_mower import context_graph +from code_mower import context_graph_command as command +from code_mower import context_graph_lifecycle as lifecycle +from code_mower import context_graph_query as query +from code_mower.context_contract import ContextError + + +NOW = datetime(2026, 1, 1, 12, tzinfo=timezone.utc) +PIN = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) + +#: Line counts the synthetic graph cites into. Every span below is inside them, +#: except where a test deliberately claims past the end of a file. +SOURCES = { + "example_pkg/config.py": 40, + "example_pkg/loader.py": 50, + "example_pkg/report.py": 20, + "tests/test_config.py": 30, +} + + +def git(repository: Path, *arguments: str) -> None: + subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + env={ + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_CONFIG_SYSTEM": "/dev/null", + "GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_NAME": "Test", "GIT_COMMITTER_EMAIL": "test@example.invalid", + "PATH": "/usr/bin:/bin:/usr/local/bin", "HOME": str(repository), + }, + ) + + +def make_repository(root: Path) -> Path: + repository = root / "checkout" + repository.mkdir() + git(repository, "init", "-q", "-b", "main") + for relative, lines in SOURCES.items(): + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(f"line {n}" for n in range(1, lines + 1)) + "\n", encoding="utf-8") + git(repository, "add", ".") + git(repository, "commit", "-q", "-m", "initial") + return repository + + +def node(identifier: str, kind: str, name: str, path: str, start=None, end=None) -> dict: + return {"id": identifier, "kind": kind, "name": name, "path": path, + "start_line": start, "end_line": end} + + +def edge(source: str, target: str, kind: str, evidence: str = "extracted") -> dict: + return {"source": source, "target": target, "kind": kind, "evidence": evidence} + + +def graph_document() -> dict: + """A small synthetic graph: a symbol, its caller, its caller's caller, a test.""" + return { + "schema": query.GRAPH_SCHEMA, + "nodes": [ + node("n-config", "symbol", "parse_config", "example_pkg/config.py", 12, 30), + node("n-load", "symbol", "load", "example_pkg/loader.py", 40, 44), + node("n-report", "symbol", "render", "example_pkg/report.py", 5, 12), + node("n-test", "test", "test_parse_config", "tests/test_config.py", 8, 26), + node("n-config-file", "file", "config.py", "example_pkg/config.py"), + ], + "edges": [ + edge("n-load", "n-config", "calls"), + edge("n-report", "n-load", "calls", "inferred"), + edge("n-test", "n-config", "tests"), + edge("n-config-file", "n-config", "defines"), + ], + } + + +def artifact(document: dict) -> bytes: + """The generation artifact: the provider's state, packed as the lifecycle packs it.""" + body = json.dumps(document).encode() + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w", format=tarfile.PAX_FORMAT) as archive: + info = tarfile.TarInfo(query.GRAPH_MEMBER) + info.size = len(body) + info.mtime = 0 + info.mode = 0o600 + archive.addfile(info, io.BytesIO(body)) + return buffer.getvalue() + + +def indexer(document: dict, *, completeness: str = lifecycle.COMPLETE): + payload = artifact(document) + + def run(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: + request.output_path.write_bytes(payload) + return lifecycle.IndexResult(completeness=completeness, indexed_files=len(SOURCES)) + + return run + + +def envelope(**overrides) -> dict: + value = { + "schema": contract.CONNECTION_SCHEMA, + "capability_version": 1, + "connection": "example-context", + "provider": "synthetic-graph", + "kind": "repository", + "generation": "generation-one", + "state": "verified", + "identity": {"repository_root": "/example/repository"}, + "repositories": ["owner/repo"], + "recipients": ["claude:builder", "codex:reviewer", "devin:builder"], + "expires_at": "2026-01-01T13:00:00Z", + "capabilities": {"search": True, "memory": False, "revision_binding": True}, + } + value.update(overrides) + return value + + +def policy(**overrides) -> dict: + value = { + "schema": contract.POLICY_SCHEMA, + "connection": "example-context", + "policy_version": "v1", + "required": True, + } + value.update(overrides) + return value + + +class GraphWorkspace(unittest.TestCase): + """A published generation over a real commit, with an injected graph.""" + + document: dict = {} + completeness = lifecycle.COMPLETE + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.addCleanup(self._directory.cleanup) + self.root = Path(self._directory.name).resolve() + self.state = self.root / "state" + self.repository = make_repository(self.root) + self.manifest = self.publish(self.document or graph_document()) + + def publish(self, document: dict, *, completeness: str | None = None): + return lifecycle.build_graph( + self.repository, + pin=PIN, + indexer=indexer(document, completeness=completeness or self.completeness), + root=self.state, + now=NOW, + ) + + def context(self, **overrides) -> query.GraphContext: + arguments = { + "question": "impact", + "target": "parse_config", + "envelope": envelope(), + "policy": policy(), + "context_repository": "owner/repo", + "work_item": "work-item-one", + "root": self.state, + "now": NOW, + } + arguments.update(overrides) + return query.graph_context(self.repository, **arguments) + + def graph(self) -> query.CodeGraph: + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + return query.read_graph(state, lifecycle.graph_status(self.repository, root=self.state)) + + def load(self, packet: dict, *, recipient: str, revision: str | None): + """Load one packet through the shared delivery contract, as a recipient does.""" + encoded = json.dumps(packet).encode() + target = self.root / "delivered-packet.json" + target.write_bytes(encoded) + target.chmod(0o600) + return contract.load_packet( + private_root=self.root, + reference={"path": "delivered-packet.json", "sha256": hashlib.sha256(encoded).hexdigest()}, + policy=policy(), + request=contract.ContextRequest("owner/repo", "work-item-one", recipient, revision), + authorize=lambda: envelope(), + now=NOW, + ) + + +class GraphSchemaTests(unittest.TestCase): + """The pinned schema is read strictly: an unreadable shape is a refusal.""" + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_reads_the_pinned_schema(self) -> None: + graph = self.load(graph_document()) + self.assertEqual(len(graph.nodes), 5) + self.assertEqual(graph.nodes["n-config"].citation, "example_pkg/config.py#L12-L30") + self.assertEqual(graph.nodes["n-config-file"].citation, "example_pkg/config.py") + + def test_rejects_another_schema(self) -> None: + document = graph_document() + document["schema"] = "graphify.native.v1" + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_unknown_node_and_edge_kinds(self) -> None: + for mutate in ( + lambda doc: doc["nodes"][0].update(kind="cluster"), + lambda doc: doc["edges"][0].update(kind="resembles"), + lambda doc: doc["edges"][0].update(evidence="guessed"), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_a_node_outside_the_indexed_checkout(self) -> None: + """A node that could never be cited must not be traversable either.""" + for path in ("/etc/passwd", "../sibling/config.py", ".git/config", ".graphify/nodes.bin"): + with self.subTest(path=path): + document = graph_document() + document["nodes"][0]["path"] = path + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_a_dangling_edge(self) -> None: + document = graph_document() + document["edges"].append(edge("n-config", "n-missing", "calls")) + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_duplicate_node_identifiers(self) -> None: + document = graph_document() + document["nodes"].append(dict(document["nodes"][0])) + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_an_inverted_line_span(self) -> None: + document = graph_document() + document["nodes"][0].update(start_line=30, end_line=12) + with self.assertRaises(ContextError): + self.load(document) + + def test_rejects_unrecognized_fields(self) -> None: + document = graph_document() + document["nodes"][0]["cluster"] = "semantic" + with self.assertRaises(ContextError): + self.load(document) + + +class TraversalTests(GraphWorkspace): + def query(self, **overrides) -> query.QueryResult: + arguments = {"question": "impact", "target": "parse_config"} + arguments.update(overrides) + return query.run_query(self.graph(), **arguments) + + def test_impact_walks_against_the_relationships(self) -> None: + result = self.query() + reached = {item.node.name for item in result.relations} + self.assertEqual(reached, {"load", "test_parse_config", "render"}) + self.assertFalse(result.truncated) + + def test_dependency_walks_along_them(self) -> None: + """``load`` depends on ``parse_config``; ``parse_config`` depends on nothing.""" + self.assertEqual({item.node.name for item in self.query(target="load", question="dependency")}, + {"parse_config"}) + self.assertEqual(self.query(question="dependency").relations, ()) + + def test_related_tests_answers_with_tests_only(self) -> None: + result = self.query(question="related_tests") + self.assertEqual([item.node.name for item in result.relations], ["test_parse_config"]) + + def test_symbol_is_a_one_hop_neighbourhood(self) -> None: + result = self.query(question="symbol") + self.assertEqual({item.depth for item in result.relations}, {1}) + self.assertEqual({item.node.name for item in result.relations}, + {"load", "test_parse_config", "config.py"}) + + def test_traversal_is_deterministic(self) -> None: + first = [item.node.id for item in self.query().relations] + second = [item.node.id for item in query.run_query( + self.graph(), question="impact", target="parse_config")] + self.assertEqual(first, second) + + def test_budget_truncates_and_says_so(self) -> None: + result = self.query(node_budget=1) + self.assertEqual(len(result.relations), 1) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_depth_bounds_the_walk(self) -> None: + """``render`` is two hops from ``parse_config`` and out of a one-hop walk.""" + self.assertNotIn("render", {item.node.name for item in self.query(depth=1).relations}) + + def test_an_unresolved_target_is_reported_rather_than_guessed(self) -> None: + result = self.query(target="no_such_symbol") + self.assertFalse(result.resolved) + self.assertEqual(result.omissions, ("unresolved_entities",)) + + def test_out_of_range_budget_and_depth_are_refused(self) -> None: + for arguments in ({"node_budget": 0}, {"node_budget": query.MAX_NODE_BUDGET + 1}, + {"depth": 0}, {"depth": query.MAX_DEPTH + 1}): + with self.subTest(arguments=arguments): + with self.assertRaises(ContextError): + self.query(**arguments) + + def test_an_unsupported_question_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.query(question="everything") + + +class CitationValidationTests(GraphWorkspace): + def validator(self) -> query.CitationValidator: + census = lifecycle.read_tracked_census(self.repository, self.manifest.commit) + return query.CitationValidator(self.repository, census) + + def test_validates_line_claims_against_the_bound_commit(self) -> None: + validator = self.validator() + self.assertTrue(validator.validate("example_pkg/config.py#L12-L30")) + self.assertTrue(validator.validate("example_pkg/config.py#L40")) + self.assertFalse(validator.validate("example_pkg/config.py#L41")) + + def test_an_untracked_path_is_never_cited(self) -> None: + (self.repository / "example_pkg" / "scratch.py").write_text("x\n", encoding="utf-8") + self.assertFalse(self.validator().validate("example_pkg/scratch.py#L1")) + + def test_working_tree_edits_do_not_change_the_verdict(self) -> None: + """The generation binds a commit; the file on disk today is not evidence.""" + (self.repository / "example_pkg" / "config.py").write_text("one line\n", encoding="utf-8") + self.assertTrue(self.validator().validate("example_pkg/config.py#L40")) + + def test_an_out_of_scope_citation_is_refused(self) -> None: + for source in ("../escape.py", "/etc/passwd", ".git/config"): + with self.subTest(source=source): + self.assertFalse(self.validator().validate(source)) + + +class PacketTests(GraphWorkspace): + def test_packet_carries_its_provenance_and_loads_through_the_contract(self) -> None: + outcome = self.context() + self.assertEqual(outcome.status, query.AVAILABLE) + packet = outcome.packet + self.assertEqual(packet["kind"], "repository") + self.assertEqual(packet["source_revision"], self.manifest.commit) + self.assertEqual(packet["source_built_at"], self.manifest.built_at) + self.assertEqual(outcome.summary["graph_generation"], self.manifest.generation) + validated = self.load(packet, recipient="claude:builder", revision=self.manifest.commit) + self.assertEqual(validated.revision_state, "matching") + + def test_every_citation_resolves_against_the_immutable_tree(self) -> None: + outcome = self.context() + report = context_graph.evaluate_graph_evidence( + outcome.packet, repository_root=self.repository, revision_state="matching", + ) + self.assertEqual(report.resolution_rate, 1.0) + self.assertTrue(report.meets_gate()) + + def test_a_citation_past_the_end_of_a_file_is_dropped_not_delivered(self) -> None: + """Evidence that cannot be pointed at is not weaker evidence; it is none.""" + document = graph_document() + document["nodes"][1].update(start_line=400, end_line=440) + self.publish(document) + outcome = self.context() + cited = {citation["source"] + for item in outcome.packet["documents"] for citation in item["citations"]} + self.assertNotIn("example_pkg/loader.py#L400-L440", cited) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + def test_confidence_maps_extracted_inferred_and_ambiguous(self) -> None: + document = graph_document() + document["edges"][2]["evidence"] = "ambiguous" + self.publish(document) + outcome = self.context() + confidences = {item["confidence"] for item in outcome.packet["documents"]} + self.assertEqual(confidences, {"extracted", "inferred", "unknown"}) + self.assertIn("unresolved_entities", outcome.packet["omissions"]) + + def test_truncation_is_reported_rather_than_hidden(self) -> None: + outcome = self.context(node_budget=1) + self.assertTrue(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "partial") + self.assertIn("provider_has_more", outcome.packet["omissions"]) + + def test_a_partial_generation_is_carried_into_the_packet(self) -> None: + """A partial build's answer is partial however complete the traversal was.""" + self.publish(graph_document(), completeness=lifecycle.PARTIAL) + outcome = self.context(policy=policy(required=False)) + # A partial generation is not usable at all by default, so the optional + # request degrades rather than delivering an answer that looks whole. + self.assertEqual(outcome.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "partial") + + def test_packet_text_carries_no_indexed_content(self) -> None: + outcome = self.context() + prose = " ".join(item["text"] for item in outcome.packet["documents"]) + self.assertNotIn("line 12", prose) + for name in ("parse_config", "impact"): + self.assertIn(name, prose) + + +class RecipientNeutralityTests(GraphWorkspace): + """One approved packet, three recipients, no provider tools or credentials.""" + + #: A packet identity, not a secret: ``render_evidence`` requires a handle. + DELIVERY = "0123456789abcdef0123456789abcdef" + + def test_claude_codex_and_devin_receive_the_same_packet(self) -> None: + packet = self.context().packet + rendered = set() + for recipient in ("claude:builder", "codex:reviewer", "devin:builder"): + with self.subTest(recipient=recipient): + validated = self.load(packet, recipient=recipient, revision=self.manifest.commit) + self.assertEqual(validated.shareable_summary()["kind"], "repository") + rendered.add(context_delivery.render_evidence(validated, self.DELIVERY)) + # The delivered payload is identical for every recipient: nothing in it + # names a connection, a credential, a provider tool, or a local path. + self.assertEqual(len(rendered), 1) + payload = rendered.pop() + self.assertNotIn("graphify", payload.casefold()) + self.assertNotIn(str(self.repository), payload) + + def test_an_unauthorized_recipient_is_still_refused(self) -> None: + with self.assertRaises(ContextError): + self.load(self.context().packet, recipient="claude:orchestrator", + revision=self.manifest.commit) + + +class AvailabilityTests(GraphWorkspace): + """Required unavailable context blocks; optional unavailable context degrades.""" + + def move_head(self) -> None: + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + + def test_stale_graph_blocks_required_context(self) -> None: + self.move_head() + outcome = self.context() + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.dependent_work, "paused") + self.assertEqual(outcome.exit_code, 1) + self.assertEqual(outcome.summary["reason"], "stale") + self.assertIsNone(outcome.packet) + + def test_stale_graph_degrades_optional_context_to_ordinary_tools(self) -> None: + self.move_head() + outcome = self.context(policy=policy(required=False)) + self.assertEqual(outcome.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(outcome.dependent_work, "usable") + self.assertEqual(outcome.exit_code, 0) + self.assertIn("ordinary repository tools", outcome.summary["next_action"]) + + def test_an_absent_graph_is_not_a_failure_of_the_caller(self) -> None: + lifecycle.remove_graph(self.repository, root=self.state) + self.assertEqual(self.context(policy=policy(required=False)).status, + query.OPTIONAL_UNAVAILABLE) + self.assertEqual(self.context().status, query.REQUIRED_UNAVAILABLE) + + def test_a_generation_without_the_pinned_document_is_unreadable(self) -> None: + self.publish({"schema": "graphify.native.v1", "nodes": [], "edges": []}) + outcome = self.context() + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "unreadable") + + def test_an_unresolved_target_blocks_required_context(self) -> None: + outcome = self.context(target="no_such_symbol") + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "unresolved") + + def test_summaries_carry_metadata_only(self) -> None: + for outcome in (self.context(), self.context(target="no_such_symbol")): + rendered = json.dumps(outcome.summary) + self.assertNotIn(str(self.repository), rendered) + self.assertNotIn(str(self.state), rendered) + + +class CommandTests(GraphWorkspace): + def authorization(self, **overrides) -> Path: + payload = { + "connection": envelope(), + "policy": policy(), + "repository": "owner/repo", + "work_item": "work-item-one", + } + payload.update(overrides) + path = self.root / "authorization.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def invoke(self, *arguments: str) -> tuple[int, dict]: + stream = io.StringIO() + with contextlib.redirect_stdout(stream): + code = command.main([ + "query", "--repo-path", str(self.repository), "--state-dir", str(self.state), + "--authorization", str(self.authorization()), "--json", *arguments, + ]) + return code, json.loads(stream.getvalue()) + + def test_query_emits_a_metadata_summary_and_writes_a_private_packet(self) -> None: + destination = self.root / "packet.json" + code, summary = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 0) + self.assertEqual(summary["status"], query.AVAILABLE) + self.assertEqual(summary["source_revision"], self.manifest.commit) + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(destination.read_text())["kind"], "repository") + # The summary is what an operator may paste anywhere; the evidence is + # only in the private file they named. + self.assertNotIn("citations", json.dumps(summary)) + + def test_required_unavailable_exits_non_zero_without_a_packet(self) -> None: + lifecycle.remove_graph(self.repository, root=self.state) + destination = self.root / "packet.json" + code, summary = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 1) + self.assertEqual(summary["status"], query.REQUIRED_UNAVAILABLE) + self.assertFalse(destination.exists()) + + def test_an_unreadable_authorization_file_fails_closed(self) -> None: + path = self.root / "authorization.json" + path.write_text(json.dumps({"connection": envelope()}), encoding="utf-8") + code = command.main([ + "query", "--repo-path", str(self.repository), "--state-dir", str(self.state), + "--authorization", str(path), "--question", "impact", "--target", "parse_config", + ]) + self.assertEqual(code, 1) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() From cc66dc385c97ced0c3d3f84bc18d1d2fd6d05484 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:07:56 -0700 Subject: [PATCH 02/33] Fix the query suite and package the new graph query seam CI ran the suite this lane's sandbox could not. Three real defects: - Two tests iterated a QueryResult rather than its relations. - The command test's authorization envelope expired at the fixed NOW the library-level tests use, but the command has no injected clock and authorizes against the real one, so every run was refused as expired. - The new module and doc were missing from the package materializer. Co-Authored-By: Claude Opus 5 (1M context) --- code-mower-package-manifest.json | 10 ++++++++++ src/code_mower/package_manifest.py | 6 ++++++ tests/test_context_graph_query.py | 14 +++++++++----- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/code-mower-package-manifest.json b/code-mower-package-manifest.json index 55c8776f..b5fdb0c7 100644 --- a/code-mower-package-manifest.json +++ b/code-mower-package-manifest.json @@ -102,6 +102,11 @@ "source": "docs/context-graph-lifecycle.md", "target": "docs/context-graph-lifecycle.md" }, + { + "kind": "doc", + "source": "docs/context-graph-queries.md", + "target": "docs/context-graph-queries.md" + }, { "kind": "doc", "source": "docs/context-packet-schema.md", @@ -707,6 +712,11 @@ "source": "src/code_mower/context_graph_lifecycle.py", "target": "src/code_mower/context_graph_lifecycle.py" }, + { + "kind": "core", + "source": "src/code_mower/context_graph_query.py", + "target": "src/code_mower/context_graph_query.py" + }, { "kind": "core", "source": "src/code_mower/context_guided.py", diff --git a/src/code_mower/package_manifest.py b/src/code_mower/package_manifest.py index d9a024ed..aa93aaa0 100644 --- a/src/code_mower/package_manifest.py +++ b/src/code_mower/package_manifest.py @@ -42,6 +42,11 @@ "src/code_mower/context_graph_lifecycle.py", "core", ), + ( + "src/code_mower/context_graph_query.py", + "src/code_mower/context_graph_query.py", + "core", + ), ( "src/code_mower/context_graph_command.py", "src/code_mower/context_graph_command.py", @@ -583,6 +588,7 @@ ("docs/slack-contract.md", "docs/slack-contract.md", "doc"), ("docs/graphify-evaluation.md", "docs/graphify-evaluation.md", "doc"), ("docs/context-graph-lifecycle.md", "docs/context-graph-lifecycle.md", "doc"), + ("docs/context-graph-queries.md", "docs/context-graph-queries.md", "doc"), ("docs/context-connections.md", "docs/context-connections.md", "doc"), ("docs/context-setup.md", "docs/context-setup.md", "doc"), ("docs/v130-release-notes.md", "docs/v130-release-notes.md", "doc"), diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 8526d30c..e27b8695 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -22,7 +22,7 @@ import tarfile import tempfile import unittest -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from pathlib import Path from code_mower import context_contract as contract @@ -291,8 +291,8 @@ def test_impact_walks_against_the_relationships(self) -> None: def test_dependency_walks_along_them(self) -> None: """``load`` depends on ``parse_config``; ``parse_config`` depends on nothing.""" - self.assertEqual({item.node.name for item in self.query(target="load", question="dependency")}, - {"parse_config"}) + reached = self.query(target="load", question="dependency").relations + self.assertEqual({item.node.name for item in reached}, {"parse_config"}) self.assertEqual(self.query(question="dependency").relations, ()) def test_related_tests_answers_with_tests_only(self) -> None: @@ -308,7 +308,7 @@ def test_symbol_is_a_one_hop_neighbourhood(self) -> None: def test_traversal_is_deterministic(self) -> None: first = [item.node.id for item in self.query().relations] second = [item.node.id for item in query.run_query( - self.graph(), question="impact", target="parse_config")] + self.graph(), question="impact", target="parse_config").relations] self.assertEqual(first, second) def test_budget_truncates_and_says_so(self) -> None: @@ -505,8 +505,12 @@ def test_summaries_carry_metadata_only(self) -> None: class CommandTests(GraphWorkspace): def authorization(self, **overrides) -> Path: + # The command has no injected clock: it authorizes against the real one, + # exactly as an operator's run does. So the envelope has to be live now + # rather than at the fixed ``NOW`` the library-level tests use. + live = datetime.now(timezone.utc) + timedelta(minutes=30) payload = { - "connection": envelope(), + "connection": envelope(expires_at=live.isoformat()), "policy": policy(), "repository": "owner/repo", "work_item": "work-item-one", From 45e041d6e0040f8f56796d4ab0a55f24c5b4da97 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:18:20 -0700 Subject: [PATCH 03/33] Graph queries: private packet replacement, true edges, seed truncation Addresses the Code Mower Codex audit at 5edb190. [P1] `--packet-out` into an existing world-readable file put the evidence behind that file's permissions: a creation mode binds only a file the open creates. The packet now goes to a freshly created private sibling and is renamed over the destination, which is also atomic. [P2] A second-hop result reported the seed as the relationship's other end, so a two-hop walk from `parse_config` through `load` to `render` asserted "render calls parse_config" and cited two nodes with no edge between them. `Relation.origin` is now the actual other endpoint of the edge that was crossed, the seed travels alongside as `Relation.seed` and reads as a "reached from" clause, and each citation is titled with the node it points at. [P2] The seed bound sliced matching definitions away silently, so a packet could report complete, untruncated evidence while never starting from some of the target's definitions. Seed overflow now sets `truncated`, raises `provider_has_more`, and counts as ambiguity. [P2] The two tests that iterated a `QueryResult` rather than its relations were already fixed at cc66dc3, as was the command test whose envelope expired at the fixed library clock. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 19 +++++- src/code_mower/context_graph_command.py | 31 ++++++++- src/code_mower/context_graph_query.py | 89 +++++++++++++++++++------ tests/test_context_graph_query.py | 55 +++++++++++++++ 4 files changed, 168 insertions(+), 26 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 341ac13e..e0db9344 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -77,7 +77,16 @@ generation and one question produce one answer, every time, and a budget cut removes the furthest relationships rather than arbitrary ones. Reaching the node budget sets `truncated` and raises `provider_has_more`. It -never silently shortens the answer. +never silently shortens the answer. A target name carried by more definitions +than the seed bound allows does the same: seeds the bound drops take their whole +reachable neighbourhood out of the answer, so that is truncation too, not a +complete result over the seeds that happened to sort first. + +Every reported relationship is the one edge the walk crossed, between that +edge's own two endpoints. A second-hop result names the intermediate node and +cites it — `render calls load (inferred, hop 2, reached from parse_config, …)` — +rather than asserting a direct relationship between the seed and the node two +hops away, which the graph does not carry. ## Citations are validated against the bound commit @@ -174,7 +183,13 @@ connection, not to a query. Standard output is metadata only — counts, states, the bound revision and generation, and the omission codes. The evidence itself goes to the private file -named by `--packet-out`, created `0600`, or nowhere at all. +named by `--packet-out`, created `0600`, or nowhere at all. A destination that +already exists is replaced rather than reopened: a creation mode binds only a +file the open creates, so writing into an existing world-readable path would put +the evidence behind whatever permissions that path already carried. The packet +is written to a freshly created private sibling and renamed over the +destination, which is also atomic — a reader never sees a half-written packet, +and a failed write leaves the previous file untouched. ## Boundary diff --git a/src/code_mower/context_graph_command.py b/src/code_mower/context_graph_command.py index 771047ad..a48595bb 100644 --- a/src/code_mower/context_graph_command.py +++ b/src/code_mower/context_graph_command.py @@ -81,11 +81,36 @@ def _write_packet(destination: Path, packet: dict) -> None: ``0o600`` at creation rather than after: the delivery path refuses a packet whose mode ever allowed anyone else, and a chmod after the fact is a window in which it did. + + A destination that already exists is *replaced*, never reopened. The + creation mode an open passes is only honoured for a file the open creates, + so writing into an existing world-readable path would put the evidence + behind whatever permissions that path already carried -- which the delivery + contract then refuses, after the bytes are already readable. The packet is + therefore written to a freshly created private sibling and renamed over the + destination, which is also atomic: a reader never sees a half-written + packet, and a failed write leaves the previous file untouched. """ body = json.dumps(packet, ensure_ascii=False, allow_nan=False, sort_keys=True, separators=(",", ":")) - descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600) - with os.fdopen(descriptor, "w", encoding="utf-8") as stream: - stream.write(body) + # ``O_EXCL`` so the mode below is the mode of a file this process created. + # The name is unique per process rather than random: this directory is the + # operator's own, and a leftover from a crashed run must not be adopted. + staging = destination.with_name(f".{destination.name}.{os.getpid()}.partial") + descriptor = os.open(staging, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + stream.write(body) + stream.flush() + os.fsync(stream.fileno()) + os.replace(staging, destination) + except BaseException: + # The evidence never survives a failed write under a name anyone asked + # for, and never under the staging name either. + try: + os.unlink(staging) + except OSError: + pass + raise def _emit(payload: dict, *, as_json: bool, text: str) -> None: diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 4e901ac9..52d6f802 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -168,19 +168,31 @@ class CodeGraph: outgoing: Mapping[str, tuple[GraphEdge, ...]] incoming: Mapping[str, tuple[GraphEdge, ...]] - def seeds(self, target: str) -> tuple[GraphNode, ...]: - """Nodes a query target names, by symbol name first and then by path. + def seed_matches(self, target: str) -> tuple[tuple[GraphNode, ...], bool]: + """The seeds a target names, and whether the seed bound dropped any. Symbol-first, as the adoption record requires: a bare name resolves to the symbols that carry it, and only a target that names no symbol at all is read as a path. Ordered by id so two runs against one generation seed identically. + + The overflow flag is not cosmetic. A name carried by more than + ``MAX_SEEDS`` definitions has definitions this traversal will never + start from, and every relationship reachable only from those is absent + from the answer. Silently slicing here would let ``run_query`` report a + complete, untruncated result over a graph it only partly read, which is + exactly the failure the adoption record's condition 4 is about. """ name = _text(target, maximum=512) matches = [node for node in self.nodes.values() if node.name == name] if not matches: matches = [node for node in self.nodes.values() if node.path == name] - return tuple(sorted(matches, key=lambda node: node.id))[:MAX_SEEDS] + ordered = tuple(sorted(matches, key=lambda node: node.id)) + return ordered[:MAX_SEEDS], len(ordered) > MAX_SEEDS + + def seeds(self, target: str) -> tuple[GraphNode, ...]: + """The bounded seed set alone, for callers that do not report truncation.""" + return self.seed_matches(target)[0] def _member(value: Any, keys: set[str], *, what: str) -> Mapping[str, Any]: @@ -324,11 +336,22 @@ def read_graph(state: lifecycle.GraphStateRoot, status: lifecycle.GenerationStat @dataclass(frozen=True) class Relation: - """One traversal result: a reached node, how it was reached, and from where.""" + """One traversal result: a reached node, the edge that reached it, and both ends. + + ``origin`` is the *other endpoint of ``via``* -- the node the walk expanded + when it found this one -- and never the seed it started from. Those differ + from the second hop onwards, and conflating them is how a traversal comes + to assert a relationship the graph does not carry: a two-hop walk from + ``parse_config`` that reaches ``render`` through ``load`` would otherwise + read as "render calls parse_config" and cite two locations that have no + edge between them. ``seed`` keeps the provenance that conflation was + standing in for, without putting it in the claim. + """ node: GraphNode via: GraphEdge origin: GraphNode + seed: GraphNode depth: int @@ -398,7 +421,7 @@ def run_query( if type(limit) is not int or not 1 <= limit <= MAX_DEPTH: raise ContextError("local graph traversal depth is out of range") direction, kinds = _TRAVERSALS[question] - seeds = graph.seeds(target) + seeds, seed_overflow = graph.seed_matches(target) omissions: list[str] = [] if not seeds: return QueryResult( @@ -407,33 +430,42 @@ def run_query( omissions=("unresolved_entities",), ) # More than one definition carries the target's name, so every relationship - # below is reported from a seed set the provider could not disambiguate. - ambiguous = len(seeds) > 1 + # below is reported from a seed set the provider could not disambiguate -- + # and a name with more definitions than the seed bound allows is the same + # uncertainty, only worse. + ambiguous = len(seeds) > 1 or seed_overflow seen = {node.id for node in seeds} relations: list[Relation] = [] - truncated = False + over_budget = False frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] while frontier: - node, origin, level = frontier.pop(0) + node, seed, level = frontier.pop(0) if level >= limit: continue for edge, other_id in _neighbours(graph, node.id, direction, kinds): if other_id in seen: continue if len(relations) >= node_budget: - truncated = True + over_budget = True break seen.add(other_id) reached = graph.nodes[other_id] - relations.append(Relation(node=reached, via=edge, origin=origin, depth=level + 1)) - frontier.append((reached, origin, level + 1)) - if truncated: + # ``node``, not ``seed``: the relationship being reported is the one + # this edge carries, between the node the walk expanded and the node + # it just reached. The seed travels alongside as provenance. + relations.append(Relation(node=reached, via=edge, origin=node, seed=seed, depth=level + 1)) + frontier.append((reached, seed, level + 1)) + if over_budget: break if question == "related_tests": # Relationship-filtered is not the same as answer-filtered: the walk # reaches callers so that a test two hops away is found, but only the # tests are the answer. relations = [item for item in relations if item.node.kind == "test"] + # Two different ways to have left something out, reported as one state: a + # relationship budget that stopped the walk, and a seed bound that stopped + # it from ever starting at some of the target's definitions. + truncated = over_budget or seed_overflow if truncated: omissions.append("provider_has_more") if ambiguous or any(item.via.evidence == "ambiguous" for item in relations): @@ -544,6 +576,11 @@ def _relation_text(question: str, item: Relation) -> str: Names, paths and relationship kinds only: everything here is already in the citations beside it, so the prose adds no claim a recipient cannot check. + + The sentence states exactly the one edge ``via`` carries, between its own + two endpoints. Where the walk reached that edge from is a separate clause, + ``reached from``, so a recipient reads a transitive result as a path and + never as a direct relationship the graph does not assert. """ verb = { "calls": "calls", "imports": "imports", "defines": "defines", @@ -553,9 +590,10 @@ def _relation_text(question: str, item: Relation) -> str: subject, object_ = item.node.name, item.origin.name else: subject, object_ = item.origin.name, item.node.name + provenance = "" if item.depth <= 1 else f", reached from {item.seed.name}" return ( f"{question}: {subject} {verb} {object_} " - f"({item.via.evidence}, hop {item.depth}, {item.node.kind} at {item.node.path})" + f"({item.via.evidence}, hop {item.depth}{provenance}, {item.node.kind} at {item.node.path})" ) @@ -586,11 +624,17 @@ def _documents( if len(documents) >= MAX_DOCUMENTS: dropped = True break - # ``dict.fromkeys`` rather than a set: a self-referential relationship - # cites one location once, in a fixed order. - candidates = list(dict.fromkeys((item.node.citation, item.origin.citation))) - cited = [candidate for candidate in candidates if validator.validate(candidate)] - if len(cited) != len(candidates): + # Both endpoints of the edge the sentence states, never the seed the + # walk started from: a citation is where a recipient goes to check the + # claim, and the claim is about these two nodes. Keyed by citation and + # first-write-wins, so a self-referential relationship cites one + # location once, in a fixed order. + endpoints: dict[str, GraphNode] = {} + for endpoint in (item.node, item.origin): + endpoints.setdefault(endpoint.citation, endpoint) + cited = [(citation, endpoint) for citation, endpoint in endpoints.items() + if validator.validate(citation)] + if len(cited) != len(endpoints): # The graph claimed a location the bound commit does not carry. # That is the provider disagreeing with the immutable tree, which # a recipient must be told about even when the relationship keeps @@ -608,9 +652,12 @@ def _documents( "text": _relation_text(result.question, item), "confidence": EVIDENCE_CONFIDENCE[item.via.evidence], "source_kind": "local_repository_graph", + # Each citation is titled with the node it actually points at, so a + # two-endpoint relationship does not label the endpoint it came + # from with the name of the one it reached. "citations": [ - {"source": source, "title": f"{item.node.kind} {item.node.name}"} - for source in cited + {"source": citation, "title": f"{endpoint.kind} {endpoint.name}"} + for citation, endpoint in cited ], }) if unvalidated: diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index e27b8695..029127bf 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -311,12 +311,35 @@ def test_traversal_is_deterministic(self) -> None: self.graph(), question="impact", target="parse_config").relations] self.assertEqual(first, second) + def test_a_second_hop_reports_the_edge_it_actually_walked(self) -> None: + """The graph says ``render`` calls ``load``, and nothing about render and parse_config.""" + reached = {item.node.name: item for item in self.query().relations} + self.assertEqual(reached["load"].origin.name, "parse_config") + self.assertEqual(reached["render"].depth, 2) + self.assertEqual(reached["render"].origin.name, "load") + # The seed the walk started from is still carried, as provenance rather + # than as a relationship anybody asserted. + self.assertEqual(reached["render"].seed.name, "parse_config") + def test_budget_truncates_and_says_so(self) -> None: result = self.query(node_budget=1) self.assertEqual(len(result.relations), 1) self.assertTrue(result.truncated) self.assertIn("provider_has_more", result.omissions) + def test_more_definitions_than_the_seed_bound_is_reported_as_truncation(self) -> None: + """Seeds the bound dropped take their whole reachable neighbourhood with them.""" + document = graph_document() + for index in range(query.MAX_SEEDS + 1): + document["nodes"].append( + node(f"n-extra-{index}", "symbol", "parse_config", "example_pkg/loader.py", 1, 2)) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + result = query.run_query(graph, question="impact", target="parse_config") + self.assertEqual(len(result.seeds), query.MAX_SEEDS) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + self.assertIn("unresolved_entities", result.omissions) + def test_depth_bounds_the_walk(self) -> None: """``render`` is two hops from ``parse_config`` and out of a one-hop walk.""" self.assertNotIn("render", {item.node.name for item in self.query(depth=1).relations}) @@ -419,6 +442,23 @@ def test_a_partial_generation_is_carried_into_the_packet(self) -> None: self.assertEqual(outcome.status, query.OPTIONAL_UNAVAILABLE) self.assertEqual(outcome.summary["reason"], "partial") + def test_multi_hop_evidence_names_and_cites_the_edge_it_walked(self) -> None: + """A transitive result reads as a path, never as a direct relationship.""" + documents = {item["text"]: item for item in self.context().packet["documents"]} + [text] = [item for item in documents if "render" in item] + self.assertIn("render calls load", text) + self.assertIn("reached from parse_config", text) + self.assertEqual( + {citation["source"] for citation in documents[text]["citations"]}, + {"example_pkg/report.py#L5-L12", "example_pkg/loader.py#L40-L44"}, + ) + # Each citation is titled with the node it points at, not with the node + # the relationship happened to reach. + self.assertEqual( + {citation["title"] for citation in documents[text]["citations"]}, + {"symbol render", "symbol load"}, + ) + def test_packet_text_carries_no_indexed_content(self) -> None: outcome = self.context() prose = " ".join(item["text"] for item in outcome.packet["documents"]) @@ -544,6 +584,21 @@ def test_query_emits_a_metadata_summary_and_writes_a_private_packet(self) -> Non # only in the private file they named. self.assertNotIn("citations", json.dumps(summary)) + def test_an_existing_readable_destination_is_replaced_by_a_private_file(self) -> None: + """A creation mode binds only a file the open created; this one replaces.""" + destination = self.root / "packet.json" + destination.write_text("stale", encoding="utf-8") + destination.chmod(0o644) + code, _ = self.invoke( + "--question", "impact", "--target", "parse_config", + "--packet-out", str(destination), + ) + self.assertEqual(code, 0) + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertEqual(json.loads(destination.read_text())["kind"], "repository") + # Nothing is left behind under a name the operator did not ask for. + self.assertEqual([item.name for item in self.root.iterdir() if "partial" in item.name], []) + def test_required_unavailable_exits_non_zero_without_a_packet(self) -> None: lifecycle.remove_graph(self.repository, root=self.state) destination = self.root / "packet.json" From fcf6a5a31d41b434ea725b63f5e92100cc13a6d0 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:31:14 -0700 Subject: [PATCH 04/33] Graph queries: read the pinned Graphify export, not an invented schema `load_graph` required exactly `schema`/`nodes`/`edges` and a `code_mower.contextGraph.v1` declaration. The lifecycle (#913) produces no such document: it archives the pinned provider's own `graph.json`, whose exporter emits a NetworkX `node_link_data` shape -- `nodes`/`links`, Graphify source-location strings, and uppercase confidence labels. So every ordinary generation was rejected by the query command, and the adapter's strictness was strictness about a shape nobody produces. Replace it with a bounded reader for the real format, validating the *provider's* contract rather than one of ours: the required node and edge fields of its validator, its `file_type` and `confidence` vocabularies, its `L` locations, and `built_at_commit` checked against the commit the generation is bound to. What the real export carries and must therefore load: exporter and extractor annotations this module does not read (`community`, `norm_label`, `confidence_score`, `weight`, `metadata`); relations outside the extractor's fixed set, since the provider's validator does not constrain `relation` and its LLM extraction emits more -- grouped as `related` so they never stand in for a `calls` claim, while the packet sentence still states the provider's own word; sourceless cross-file stubs, traversable and never citable; and non-code corpora, dropped with their links pruned as the exporter's own `prune_dangling_edges` does. Node kinds are now derived in `_node_kind` from the shape the pinned extractor emits, and named as a derivation, because the export states no such kind. Citations are one line per node rather than a span: `source_location` records no extent. Reference: Graphify-Labs/graphify at 23f2ffa (release 0.9.58), `graphify/export.py`, `graphify/validate.py`, `graphify/extractors/engine.py`. Not executed locally: no interpreter in this lane's sandbox can run a `requires-python >= 3.12` codebase. CI is the first execution. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 78 ++++-- src/code_mower/context_graph_query.py | 387 ++++++++++++++++++++------ tests/test_context_graph_query.py | 246 ++++++++++++---- 3 files changed, 560 insertions(+), 151 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index e0db9344..1ac3234e 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -18,32 +18,78 @@ Nothing here installs, downloads, or runs a provider, and nothing here is on a default path. The only subprocess is Git, reading blobs of the commit the generation is already bound to. -## Reading the pinned schema directly +## Reading the pinned provider's own export A generation's artifact is the provider's own state, packed reproducibly. This -adapter reads one member of it, `graph.json`, and requires it to declare -`code_mower.contextGraph.v1`: +adapter reads one member of it, `graph.json` — the file the pinned Graphify +release writes from `graphify/export.py::to_json`. There is no Code Mower graph +schema and no normalization pass between the build and the query: the lifecycle +archives what the provider wrote, so this is what gets read. ```json { - "schema": "code_mower.contextGraph.v1", + "directed": false, "multigraph": false, "graph": {}, "nodes": [ - {"id": "n-config", "kind": "symbol", "name": "parse_config", - "path": "example_pkg/config.py", "start_line": 12, "end_line": 30} + {"id": "n-config", "label": "parse_config", "file_type": "code", + "source_file": "example_pkg/config.py", "source_location": "L12", + "community": 0, "norm_label": "parse_config"} ], - "edges": [ - {"source": "n-load", "target": "n-config", "kind": "calls", - "evidence": "extracted"} - ] + "links": [ + {"source": "n-load", "target": "n-config", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "example_pkg/loader.py", + "source_location": "L41", "weight": 1.0, "confidence_score": 1.0} + ], + "hyperedges": [], + "built_at_commit": "…" } ``` -Node kinds are `file`, `symbol`, `test`. Edge kinds are `calls`, `imports`, -`defines`, `references`, `tests`. Every other shape is a refusal, not a -best-effort read: an adapter that repairs what it does not understand reports a -traversal over a graph nobody reviewed. The refusals are exhaustive on purpose — -an unknown kind, a dangling edge, a duplicate identifier, an inverted line span, -an unrecognized field, a node path that escapes the indexed checkout. +What is validated is the *provider's* contract, not one of ours: + +- The required node and edge fields of `graphify/validate.py` — `id`, `label`, + `file_type`, `source_file` on a node; `source`, `target`, `relation`, + `confidence`, `source_file` on a link. A record missing one would not have + passed the provider's own validator, so it is a refusal here. +- Its vocabularies. `file_type` must be one of the six it defines, and + `confidence` must be uppercase `EXTRACTED`/`INFERRED`/`AMBIGUOUS`. Lowercase + is the *packet* vocabulary, and a graph using it was not written by the + pinned exporter. +- Its locations. `source_location` is `L` or empty; anything else is a + location this module could not check against the bound commit, so it refuses + rather than traversing past it. One line per node, never a span: the export + records no extent, and claiming one would be this adapter inventing it. +- `built_at_commit`, when the exporter stamped it, must equal the commit the + generation is bound to. Otherwise the artifact and the manifest describe + different revisions. + +Three things are deliberately *not* refusals, because the real export carries +them and rejecting them would reject every ordinary generation: + +- **Extra annotations.** The exporter adds `community`, `community_name` and + `norm_label` to nodes and `confidence_score` to links; the extractor adds + `weight`, `context`, `type` and a free-form `metadata` dict from an LLM + extraction. None of them changes a traversal, so none is read. Everything + this module *does* read is read by name and bounded. +- **Relations outside the mapped set.** The provider's validator does not + constrain `relation` at all. Mapped relations (`calls`, `imports`, `defines`, + `contains`, `references`, `inherits`, `implements`, `tests`) decide which + traversals an edge participates in; anything else is grouped as `related`, + reachable only from the `symbol` neighbourhood. Either way the sentence in + the packet states the provider's own word, so an `implements` edge reads as + "implements" and a `supersedes` edge reads as "supersedes". +- **Sourceless stubs and non-code corpora.** The extractor emits nodes with an + empty `source_file` for cross-file references it could not resolve; those + stay traversable and are never cited. Nodes whose `file_type` is not `code` + are dropped, and links onto a dropped node are pruned — which is the pinned + exporter's own treatment in `prune_dangling_edges`. + +Node kinds — `file`, `symbol`, `test` — are **derived**, not read: a Graphify +node declares its corpus and, rarely, a `type`, but never whether it is a file, +a definition, or a test. A file node is the one the extractor emits per file, +whose label is that file's base name; a test is a code node whose path sits in +this repository's test layout; everything else is a symbol. That derivation is +the one place this adapter infers something the provider did not state, and it +is named in `_node_kind` for that reason. Reading the member directly, rather than through provider query tooling, is what makes the traversal reproducible and the bounds ours. It also means a recipient diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 52d6f802..db8d7d61 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -56,16 +56,78 @@ ) from .context_graph import MAX_GRAPH_CITATIONS, parse_graph_citation -#: The graph document Code Mower reads, by name and by schema. Pinned on both: -#: the artifact is whatever the pinned provider release wrote, and a member -#: that is merely *shaped* like a graph is not the schema this adapter was -#: reviewed against. An artifact without it is unreadable rather than -#: best-effort -- see ``read_graph``. +#: The graph document Code Mower reads. This is the pinned provider's own +#: ``graph.json`` -- the file ``graphify/export.py::to_json`` writes at commit +#: ``23f2ffa`` (release 0.9.58), which is the release the lifecycle (#913) pins +#: and archives. There is no Code Mower graph schema and no normalization step +#: between the two: an adapter that required a shape the build never produces +#: would reject every real generation, so this module reads the provider's +#: actual export and does the narrowing itself. GRAPH_MEMBER = "graph.json" -GRAPH_SCHEMA = "code_mower.contextGraph.v1" QUERY_SCHEMA = "code_mower.contextGraphQuery.v1" +#: The provider's export is a NetworkX ``node_link_data`` document: ``nodes`` +#: plus ``links``. ``edges`` is the same list under the name NetworkX used +#: before 3.2, and the pinned validator accepts either, so this reader does +#: too. Other top-level keys the pinned exporter writes -- ``directed``, +#: ``multigraph``, ``graph``, ``hyperedges``, ``built_at_commit`` -- are +#: provider bookkeeping; only ``built_at_commit`` carries a claim this module +#: acts on, and it is checked against the generation rather than trusted. +GRAPH_EDGE_KEYS = ("links", "edges") + +#: Required node and edge fields, taken from the pinned validator's +#: ``REQUIRED_NODE_FIELDS`` and ``REQUIRED_EDGE_FIELDS``. A record missing one +#: of these is a refusal: the provider's own validator would not have passed +#: it, so a graph carrying it was not produced by the build this adapter was +#: reviewed against. +GRAPH_NODE_FIELDS = ("id", "label", "file_type", "source_file") +GRAPH_EDGE_FIELDS = ("source", "target", "relation", "confidence", "source_file") + +#: The pinned validator's ``VALID_FILE_TYPES``. Only ``code`` is traversable +#: here -- the others are the provider's document/paper/image/rationale/concept +#: corpora, which are not the repository relationships #914 is about and carry +#: no line-checkable location in the bound commit. +GRAPH_FILE_TYPES = frozenset({"code", "document", "paper", "image", "rationale", "concept"}) +CODE_FILE_TYPE = "code" + +#: The pinned validator's ``VALID_CONFIDENCES``, lowercased. The provider +#: writes these uppercase; the packet contract's vocabulary is lowercase, and +#: this is the whole of the difference. +GRAPH_CONFIDENCES = {"EXTRACTED": "extracted", "INFERRED": "inferred", "AMBIGUOUS": "ambiguous"} + +#: A node's location in the pinned export is ``source_location``, a string of +#: the form ``L`` written by the extractor's ``add_node``/``add_edge``. +#: Cross-file stubs carry ``""`` -- a real node with no location, which this +#: module keeps traversable and refuses to cite. +_LOCATION_MAX_LINE = 10_000_000 + +#: The pinned extractor's relation vocabulary, normalized onto the +#: relationships a query filters by. ``contains`` is the extractor's +#: file-to-definition and class-to-member edge, which is what ``defines`` +#: means here; ``implements`` is inheritance by another name. The provider's +#: own word is kept on the edge and is what a packet sentence states -- this +#: mapping only decides which traversals an edge participates in. +#: +#: Relations outside this table are *not* a refusal. The pinned validator does +#: not constrain ``relation`` at all, and the provider's LLM extraction emits +#: relations beyond the extractor's fixed set. An unmapped relation is grouped +#: as ``related``: it is carried, it is citable, and it is reachable by the +#: ``symbol`` neighbourhood, but it never stands in for a ``calls`` or an +#: ``imports`` claim it was not. +GRAPH_RELATIONS = { + "calls": "calls", + "imports": "imports", + "defines": "defines", + "contains": "defines", + "references": "references", + "inherits": "inherits", + "implements": "inherits", + "tests": "tests", +} +OTHER_RELATION = "related" +RELATION_KINDS = frozenset({*GRAPH_RELATIONS.values(), OTHER_RELATION}) + #: The questions a code graph answers better than ``rg`` and an ordinary #: reading of the tree, from the comparison set in the adoption record. Each #: names one direction and one relationship filter; there is no free-form @@ -73,12 +135,14 @@ #: cannot be bounded or reproduced. QUESTIONS = ("impact", "dependency", "symbol", "related_tests") -#: Node kinds and edge kinds of the pinned schema. An unrecognized kind is a -#: refusal, not a node this adapter quietly ignores: a graph that carries -#: relationships this code does not model would have its traversals silently -#: truncated by the model rather than by a budget anyone reported. +#: The node kinds a *query* is expressed in. The pinned export does not carry +#: them: a Graphify node declares ``file_type`` (its corpus) and, for a +#: handful of constructs, ``type`` (e.g. ``namespace``) -- never whether it is +#: a file, a definition, or a test. So these are derived, in ``_node_kind``, +#: from the shape the pinned extractor actually emits, and the derivation is +#: named here rather than hidden because it is the one place this adapter +#: infers something the provider did not say. NODE_KINDS = frozenset({"file", "symbol", "test"}) -EDGE_KINDS = frozenset({"calls", "imports", "defines", "references", "tests"}) #: How the provider's own qualification of a relationship maps onto the packet #: contract's confidence vocabulary. ``ambiguous`` is the important one: the @@ -88,13 +152,22 @@ #: and also raises the ``unresolved_entities`` omission on the packet. EVIDENCE_CONFIDENCE = {"extracted": "extracted", "inferred": "inferred", "ambiguous": "unknown"} +#: Path segments that make a code file a test in this repository's own layout. +#: A derivation, like ``_node_kind`` itself, and deliberately conservative: +#: naming a non-test file a test would put it in a ``related_tests`` answer. +_TEST_PREFIXES = ("tests/", "test/") +_TEST_STEMS = ("test_", "_test", ".test", "_spec", ".spec") + #: Relationship filters per question, and whether the traversal runs along #: edges or against them. ``impact`` asks who is affected by a change, which is -#: the reverse of ``dependency`` over the same relationships. +#: the reverse of ``dependency`` over the same relationships. The filters are +#: written in the normalized vocabulary of ``GRAPH_RELATIONS``, so an +#: ``implements`` edge participates wherever ``inherits`` does and an unmapped +#: relation participates only in the ``symbol`` neighbourhood. _TRAVERSALS: dict[str, tuple[str, frozenset[str]]] = { - "impact": ("incoming", frozenset({"calls", "imports", "references", "tests"})), - "dependency": ("outgoing", frozenset({"calls", "imports", "references"})), - "symbol": ("both", frozenset({"defines", "calls", "imports", "references", "tests"})), + "impact": ("incoming", frozenset({"calls", "imports", "references", "tests", "inherits"})), + "dependency": ("outgoing", frozenset({"calls", "imports", "references", "inherits"})), + "symbol": ("both", RELATION_KINDS), "related_tests": ("incoming", frozenset({"tests", "calls", "references"})), } @@ -128,31 +201,51 @@ @dataclass(frozen=True) class GraphNode: - """One node of the pinned schema, already held to the citation scope rules.""" + """One node of the pinned export, narrowed and held to the citation rules. + + ``path`` is the export's ``source_file`` and ``line`` is its + ``source_location``. Both may be absent: the pinned extractor emits + *sourceless stubs* for cross-file references it could not resolve locally + (``source_file`` and ``source_location`` set to ``""``), so that a + corpus-level pass can collapse them onto a real definition. Those nodes are + real relationships and stay traversable; they are simply not citable, and + ``citation`` is ``None`` for them rather than a path that points nowhere. + """ id: str kind: str name: str path: str - start_line: int | None - end_line: int | None + line: int | None @property - def citation(self) -> str: - """The node's location as a citation string, with its line span if it has one.""" - if self.start_line is None: + def citation(self) -> str | None: + """The node's location as a citation, or ``None`` if it has no location. + + The pinned export records a single line per node, not a span, so a + located node cites one line. Claiming a span the provider never stated + would be this adapter inventing the extent of a definition. + """ + if not self.path: + return None + if self.line is None: return self.path - if self.end_line is None or self.end_line == self.start_line: - return f"{self.path}#L{self.start_line}" - return f"{self.path}#L{self.start_line}-L{self.end_line}" + return f"{self.path}#L{self.line}" @dataclass(frozen=True) class GraphEdge: - """One relationship, with the provider's own qualification of it.""" + """One relationship, with the provider's own word for it and its own caveat. + + ``relation`` is what the provider wrote; ``kind`` is that relation + normalized onto ``GRAPH_RELATIONS`` for filtering. A packet sentence states + ``relation``, so a recipient reads the provider's claim and not this + module's grouping of it. + """ source: str target: str + relation: str kind: str evidence: str @@ -195,58 +288,160 @@ def seeds(self, target: str) -> tuple[GraphNode, ...]: return self.seed_matches(target)[0] -def _member(value: Any, keys: set[str], *, what: str) -> Mapping[str, Any]: - if not isinstance(value, Mapping) or set(value) != keys: - raise ContextError(f"local graph {what} fields are missing or unrecognized") +def _required(value: Any, fields: Sequence[str], *, what: str) -> Mapping[str, Any]: + """A provider record with every field its own validator requires. + + Required fields only. Unrecognized *extra* keys are not a refusal here, + which is a deliberate change from reading an invented schema: the pinned + exporter already annotates nodes with ``community``, ``community_name`` and + ``norm_label``, edges with ``confidence_score`` and ``weight``, and either + with a free-form ``metadata`` dict whose contents come from an LLM + extraction. None of those change a traversal. Refusing them would reject + every real generation -- which is exactly what a strict reader of a + hand-written schema did. + + What *is* strict: every field this module reads is read by name, bounded, + and validated against the pinned vocabulary. Nothing else is looked at. + """ + if not isinstance(value, Mapping): + raise ContextError(f"local graph {what} must be an object") + missing = [field_name for field_name in fields if field_name not in value] + if missing: + raise ContextError(f"local graph {what} is missing required provider fields") return value -def _line(value: Any) -> int | None: - if value is None: +def _maybe_text(value: Any, *, maximum: int) -> str: + """Bounded single-line text, or ``""`` for a field the provider left empty. + + ``_text`` rejects the empty string, which is correct for every identifier + in the packet contract and wrong for exactly one field here: a sourceless + stub's ``source_file``. Everything non-empty goes through ``_text`` + unchanged, so the bound and the control-character rules are the same ones. + """ + if value is None or value == "": + return "" + return _text(value, maximum=maximum) + + +def _location(value: Any) -> int | None: + """Parse the pinned export's ``source_location``: ``L``, or nothing. + + The extractor writes ``f"L{line}"``; a sourceless stub writes ``""``. A + value in any other shape is a refusal rather than a node with an unknown + location, because a location this module cannot read is one it cannot + check against the bound commit. + """ + if value is None or value == "": return None - if type(value) is not int or not 1 <= value <= 10_000_000: + text = _text(value, maximum=32) + if not text.startswith("L") or not text[1:].isdigit(): + raise ContextError("local graph source location is not a supported provider location") + line = int(text[1:]) + if not 1 <= line <= _LOCATION_MAX_LINE: raise ContextError("local graph line number is out of range") - return value + return line -def _node(value: Any) -> GraphNode: - record = _member(value, {"id", "kind", "name", "path", "start_line", "end_line"}, what="node") - kind = record["kind"] - if kind not in NODE_KINDS: - raise ContextError("unsupported local graph node kind") - start = _line(record["start_line"]) - end = _line(record["end_line"]) - if start is None and end is not None: - raise ContextError("local graph line span must start before it ends") - if start is not None and end is not None and end < start: - raise ContextError("local graph line span must start before it ends") - # Held to the citation rules here, at parse time, rather than when a packet - # is written: a node that could never be cited inside the indexed checkout - # must not be traversable either, or an out-of-scope path reaches a - # recipient as a relationship whose citation was quietly dropped. +def _node_kind(path: str, label: str, node_type: Any) -> str: + """Derive a query-level node kind from what the pinned export does carry. + + The provider states no such kind, so this reads the shape its extractor + emits: + + * A **file** node is the one the extractor creates per file, whose label is + that file's base name at ``L1`` (``add_node(_make_id(str(path)), + path.name, 1)``). Matching on the base name is what distinguishes it from + a definition inside the same file. + * A **test** is a code node whose file sits in this repository's test + layout. A path convention, not a provider claim -- ``related_tests`` + answers from it, so it is kept narrow. + * Everything else is a **symbol**: a definition, a member, a namespace, or + an unresolved cross-file stub. + """ + if node_type == "namespace": + return "symbol" + if path: + base = path.rsplit("/", 1)[-1] + lowered = path.lower() + stem = base.lower().rsplit(".", 1)[0] + is_test = ( + lowered.startswith(_TEST_PREFIXES) + or "/tests/" in lowered + or "/test/" in lowered + or stem.startswith("test_") + or stem.endswith(("_test", "_spec")) + ) + if is_test: + return "test" + if base == label: + return "file" + return "symbol" + + +def _node(value: Any) -> GraphNode | None: + """One pinned-export node, or ``None`` for a corpus this module does not query. + + A non-``code`` node is dropped rather than refused. The provider indexes + documents, papers, images, rationales and concepts into the same graph, and + those are not repository relationships: they carry no location in the bound + commit, so no traversal here could cite one. Dropping them is bounded and + visible -- every edge that named one becomes a dangling edge, which + ``load_graph`` prunes and counts. + """ + record = _required(value, GRAPH_NODE_FIELDS, what="node") + file_type = record["file_type"] + if file_type not in GRAPH_FILE_TYPES: + raise ContextError("unsupported local graph node file type") + if file_type != CODE_FILE_TYPE: + return None + path = _maybe_text(record["source_file"], maximum=1024) + label = _text(record["label"], maximum=512) node = GraphNode( id=_text(record["id"], maximum=512), - kind=kind, - name=_text(record["name"], maximum=512), - path=_text(record["path"], maximum=1024), - start_line=start, - end_line=end, + kind=_node_kind(path, label, record.get("type")), + name=label, + path=path, + line=_location(record.get("source_location")), ) - parse_graph_citation(node.citation) + # Held to the citation rules here, at parse time, rather than when a packet + # is written: a node that could never be cited inside the indexed checkout + # must not be traversable either, or an out-of-scope path reaches a + # recipient as a relationship whose citation was quietly dropped. A + # sourceless stub has nothing to hold to the rules and is exempt. + if node.citation is not None: + parse_graph_citation(node.citation) return node -def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge: - record = _member(value, {"source", "target", "kind", "evidence"}, what="edge") - if record["kind"] not in EDGE_KINDS: - raise ContextError("unsupported local graph edge kind") - if record["evidence"] not in EVIDENCE_CONFIDENCE: - raise ContextError("unsupported local graph edge evidence") +def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge | None: + """One pinned-export link, or ``None`` if either endpoint is not in the graph. + + Pruned rather than refused, which is the pinned exporter's own treatment: + ``export.py::prune_dangling_edges`` drops links whose endpoints are not in + the node set and reports a count. Two things reach this path in a real + export -- a link onto a corpus ``_node`` dropped above, and a link the + provider emitted onto an id its node list does not carry -- and neither is + a relationship this module can cite, since a citation needs a node with a + location. Dropping the edge is what makes the relationship absent from the + answer instead of present with one end unstated. + """ + record = _required(value, GRAPH_EDGE_FIELDS, what="edge") + confidence = record["confidence"] + if confidence not in GRAPH_CONFIDENCES: + raise ContextError("unsupported local graph edge confidence") + relation = _text(record["relation"], maximum=128) source = _text(record["source"], maximum=512) target = _text(record["target"], maximum=512) if source not in nodes or target not in nodes: - raise ContextError("local graph edge names a node the graph does not carry") - return GraphEdge(source=source, target=target, kind=record["kind"], evidence=record["evidence"]) + return None + return GraphEdge( + source=source, + target=target, + relation=relation, + kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), + evidence=GRAPH_CONFIDENCES[confidence], + ) def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdge, ...]]: @@ -261,29 +456,50 @@ def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdg def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> CodeGraph: - """Validate the pinned graph schema. Every unreadable shape is a refusal. - - Read strictly for the same reason the manifest is: this document is - provider output, and an adapter that repairs what it does not understand - reports a traversal over a graph nobody reviewed. + """Read the pinned provider's ``graph.json`` into a bounded queryable graph. + + The document is the pinned exporter's output, so what is validated here is + the provider's own contract -- the required fields of its validator, its + ``file_type`` and ``confidence`` vocabularies, its ``L`` locations -- + and not a shape Code Mower invented. Every value this module reads is read + by name and bounded; every value it does not read is left alone. + + Two provenance checks are worth more than any field check. The first is + ``built_at_commit``: the exporter stamps the commit the graph was built + from, and if that disagrees with the commit the generation is bound to then + the artifact and the manifest describe different revisions, which is a + refusal no traversal should be run past. The second is the census check + every citation goes through later. """ - document = _member(payload, {"schema", "nodes", "edges"}, what="document") - if document["schema"] != GRAPH_SCHEMA: - raise ContextError("unsupported local graph schema") - raw_nodes = document["nodes"] - raw_edges = document["edges"] + if not isinstance(payload, Mapping): + raise ContextError("local graph document must be an object") + raw_nodes = payload.get("nodes") + raw_edges = next( + (payload[key] for key in GRAPH_EDGE_KEYS if key in payload), + None, + ) + if raw_edges is None or raw_nodes is None: + raise ContextError("local graph document carries no provider nodes and links") if not isinstance(raw_nodes, list) or len(raw_nodes) > MAX_NODES: raise ContextError("local graph node count exceeds its budget") if not isinstance(raw_edges, list) or len(raw_edges) > MAX_EDGES: raise ContextError("local graph edge count exceeds its budget") + stamped = payload.get("built_at_commit") + if stamped is not None and _text(stamped, maximum=64) != commit: + raise ContextError("local graph was built from a different commit than its generation") nodes: dict[str, GraphNode] = {} for value in raw_nodes: node = _node(value) + if node is None: + continue if node.id in nodes: + # The provider's own validator does not check this, but its graph + # is a NetworkX node set and cannot hold two nodes under one id. A + # document that does was not written by the pinned exporter. raise ContextError("local graph node identifiers must be unique") nodes[node.id] = node edges = tuple(sorted( - (_edge(value, nodes) for value in raw_edges), + (parsed for value in raw_edges if (parsed := _edge(value, nodes)) is not None), key=lambda edge: (edge.kind, edge.source, edge.target), )) return CodeGraph( @@ -582,10 +798,11 @@ def _relation_text(question: str, item: Relation) -> str: ``reached from``, so a recipient reads a transitive result as a path and never as a direct relationship the graph does not assert. """ - verb = { - "calls": "calls", "imports": "imports", "defines": "defines", - "references": "references", "tests": "tests", - }[item.via.kind] + # The provider's own word for the relationship, not this module's grouping + # of it: an ``implements`` edge is filtered as ``inherits`` but must read as + # "implements", and an LLM-extracted relation outside the mapped set must + # read as itself rather than as the ``related`` bucket it was filed under. + verb = item.via.relation if item.via.source == item.node.id: subject, object_ = item.node.name, item.origin.name else: @@ -629,12 +846,22 @@ def _documents( # claim, and the claim is about these two nodes. Keyed by citation and # first-write-wins, so a self-referential relationship cites one # location once, in a fixed order. + # A sourceless stub contributes no citation at all -- it is an + # unresolved cross-file reference, which is the one endpoint shape the + # pinned extractor emits with no location to point at. It counts as an + # endpoint the bound commit does not confirm, same as a path the census + # does not carry. endpoints: dict[str, GraphNode] = {} + located = 0 for endpoint in (item.node, item.origin): - endpoints.setdefault(endpoint.citation, endpoint) + citation = endpoint.citation + if citation is None: + continue + located += 1 + endpoints.setdefault(citation, endpoint) cited = [(citation, endpoint) for citation, endpoint in endpoints.items() if validator.validate(citation)] - if len(cited) != len(endpoints): + if len(cited) != len(endpoints) or located < 2: # The graph claimed a location the bound commit does not carry. # That is the provider disagreeing with the immutable tree, which # a recipient must be told about even when the relationship keeps @@ -868,8 +1095,10 @@ def graph_context( "CodeGraph", "DEFAULT_NODE_BUDGET", "EVIDENCE_CONFIDENCE", + "GRAPH_CONFIDENCES", + "GRAPH_FILE_TYPES", "GRAPH_MEMBER", - "GRAPH_SCHEMA", + "GRAPH_RELATIONS", "GraphContext", "GraphEdge", "GraphNode", diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 029127bf..16ee2165 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -74,32 +74,82 @@ def make_repository(root: Path) -> Path: return repository -def node(identifier: str, kind: str, name: str, path: str, start=None, end=None) -> dict: - return {"id": identifier, "kind": kind, "name": name, "path": path, - "start_line": start, "end_line": end} +def node(identifier: str, label: str, path: str, line=None, **extra) -> dict: + """One node in the pinned exporter's own shape. + + The field names and the annotations are the ones Graphify writes at the + pinned commit: the four required fields of ``graphify/validate.py`` + (``id``, ``label``, ``file_type``, ``source_file``), the extractor's + ``source_location`` of the form ``L``, and the ``community`` / + ``community_name`` / ``norm_label`` annotations ``export.py::to_json`` + adds to every node on the way out. No Code Mower node ``kind`` and no + line span: neither exists in the real export. + """ + return { + "id": identifier, + "label": label, + "file_type": "code", + "source_file": path, + "source_location": "" if line is None else f"L{line}", + "community": 0, + "community_name": "Community 0", + "norm_label": label.lower(), + **extra, + } + +def edge(source: str, target: str, relation: str, confidence: str = "EXTRACTED", **extra) -> dict: + """One link in the pinned exporter's own shape. + + The five required edge fields, the uppercase ``confidence`` vocabulary of + the pinned validator, and the ``weight`` / ``confidence_score`` the + extractor and exporter attach. ``confidence_score`` uses the exporter's + own ``_CONFIDENCE_SCORE_DEFAULTS``. + """ + scores = {"EXTRACTED": 1.0, "INFERRED": 0.55, "AMBIGUOUS": 0.2} + return { + "source": source, + "target": target, + "relation": relation, + "confidence": confidence, + "source_file": "example_pkg/config.py", + "source_location": "L12", + "weight": 1.0, + "confidence_score": scores[confidence], + **extra, + } -def edge(source: str, target: str, kind: str, evidence: str = "extracted") -> dict: - return {"source": source, "target": target, "kind": kind, "evidence": evidence} +def graph_document(**extra) -> dict: + """A small graph in the pinned provider's export format. -def graph_document() -> dict: - """A small synthetic graph: a symbol, its caller, its caller's caller, a test.""" + Top-level shape is ``networkx.json_graph.node_link_data(G, edges="links")`` + as ``export.py::to_json`` writes it: ``directed``, ``multigraph``, + ``graph``, ``nodes``, ``links``, plus the ``hyperedges`` list and the + ``built_at_commit`` stamp the exporter appends. Contents are a symbol, its + caller, its caller's caller, a test, and the file node the extractor emits + for each indexed file. + """ return { - "schema": query.GRAPH_SCHEMA, + "directed": False, + "multigraph": False, + "graph": {}, "nodes": [ - node("n-config", "symbol", "parse_config", "example_pkg/config.py", 12, 30), - node("n-load", "symbol", "load", "example_pkg/loader.py", 40, 44), - node("n-report", "symbol", "render", "example_pkg/report.py", 5, 12), - node("n-test", "test", "test_parse_config", "tests/test_config.py", 8, 26), - node("n-config-file", "file", "config.py", "example_pkg/config.py"), + node("n-config", "parse_config", "example_pkg/config.py", 12), + node("n-load", "load", "example_pkg/loader.py", 40), + node("n-report", "render", "example_pkg/report.py", 5), + node("n-test", "test_parse_config", "tests/test_config.py", 8), + # The extractor's per-file node: label is the file's base name at L1. + node("n-config-file", "config.py", "example_pkg/config.py", 1), ], - "edges": [ + "links": [ edge("n-load", "n-config", "calls"), - edge("n-report", "n-load", "calls", "inferred"), + edge("n-report", "n-load", "calls", "INFERRED"), edge("n-test", "n-config", "tests"), - edge("n-config-file", "n-config", "defines"), + edge("n-config-file", "n-config", "contains"), ], + "hyperedges": [], + **extra, } @@ -214,28 +264,121 @@ def load(self, packet: dict, *, recipient: str, revision: str | None): class GraphSchemaTests(unittest.TestCase): - """The pinned schema is read strictly: an unreadable shape is a refusal.""" + """The pinned provider's own export is what gets read, and read bounded. + + Every fixture in here is the shape ``graphify/export.py::to_json`` writes + at the pinned commit. The tests split into two halves on purpose: what the + real export carries must load, and what the provider's own validator would + reject must refuse. + """ def load(self, document: dict) -> query.CodeGraph: return query.load_graph(document, generation="a" * 32, commit="b" * 40) - def test_reads_the_pinned_schema(self) -> None: + def test_reads_the_pinned_provider_export(self) -> None: graph = self.load(graph_document()) self.assertEqual(len(graph.nodes), 5) - self.assertEqual(graph.nodes["n-config"].citation, "example_pkg/config.py#L12-L30") - self.assertEqual(graph.nodes["n-config-file"].citation, "example_pkg/config.py") + # One line per node, because ``source_location`` records one line. + self.assertEqual(graph.nodes["n-config"].citation, "example_pkg/config.py#L12") + self.assertEqual( + {node.id: node.kind for node in graph.nodes.values()}, + { + "n-config": "symbol", "n-load": "symbol", "n-report": "symbol", + # Derived: label equals the file's base name. + "n-config-file": "file", + # Derived: the repository's test layout. + "n-test": "test", + }, + ) - def test_rejects_another_schema(self) -> None: + def test_keeps_the_providers_own_relation_and_lowercases_confidence(self) -> None: + """``contains`` filters as ``defines`` but still reads as ``contains``.""" + graph = self.load(graph_document()) + by_pair = {(edge.source, edge.target): edge for edge in graph.edges} + contains = by_pair[("n-config-file", "n-config")] + self.assertEqual((contains.relation, contains.kind), ("contains", "defines")) + self.assertEqual(by_pair[("n-report", "n-load")].evidence, "inferred") + + def test_reads_the_pre_3_2_edges_key(self) -> None: + """The pinned validator accepts ``edges`` for ``links``; so does this.""" document = graph_document() - document["schema"] = "graphify.native.v1" + document["edges"] = document.pop("links") + self.assertEqual(len(self.load(document).edges), 4) + + def test_maps_an_unlisted_relation_without_asserting_a_listed_one(self) -> None: + """An LLM-extracted relation is carried, grouped as ``related``, never renamed.""" + document = graph_document() + document["links"].append(edge("n-config", "n-report", "supersedes")) + graph = self.load(document) + extra = next(edge for edge in graph.edges if edge.relation == "supersedes") + self.assertEqual(extra.kind, query.OTHER_RELATION) + # ``related`` is not in the impact filter, so it cannot stand in for a call. + self.assertNotIn(query.OTHER_RELATION, query._TRAVERSALS["impact"][1]) + + def test_keeps_a_sourceless_stub_traversable_and_uncitable(self) -> None: + """The extractor's cross-file stub: a real node with no location.""" + document = graph_document() + document["nodes"].append({ + "id": "n-stub", "label": "Thing", "file_type": "code", + "source_file": "", "source_location": "", "origin_file": "example_pkg/config.py", + }) + document["links"].append(edge("n-config", "n-stub", "references")) + graph = self.load(document) + self.assertIsNone(graph.nodes["n-stub"].citation) + self.assertEqual(len(graph.edges), 5) + + def test_drops_non_code_corpora_and_prunes_their_edges(self) -> None: + """Documents and concepts are not repository relationships.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["links"].append(edge("n-config", "n-doc", "references")) + graph = self.load(document) + self.assertNotIn("n-doc", graph.nodes) + self.assertEqual(len(graph.edges), 4) + + def test_tolerates_provider_annotations_it_does_not_read(self) -> None: + """Extra exporter and LLM metadata must not reject a real generation.""" + document = graph_document() + document["nodes"][0]["metadata"] = {"namespace": "example_pkg", "scope_chain": ["mod"]} + document["nodes"][0]["type"] = "namespace" + document["links"][0]["context"] = "call site" + self.assertEqual(len(self.load(document).nodes), 5) + + def test_refuses_a_document_with_no_provider_nodes_and_links(self) -> None: + for document in ({"nodes": []}, {"links": []}, {"schema": "something.else"}, []): + with self.subTest(document=document): + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_a_graph_built_from_another_commit(self) -> None: + """``built_at_commit`` disagreeing with the generation is a refusal.""" with self.assertRaises(ContextError): - self.load(document) + self.load(graph_document(built_at_commit="c" * 40)) + # Agreeing is fine, and is the ordinary case. + self.assertEqual(len(self.load(graph_document(built_at_commit="b" * 40)).nodes), 5) + + def test_refuses_records_missing_the_providers_required_fields(self) -> None: + for mutate in ( + lambda doc: doc["nodes"][0].pop("label"), + lambda doc: doc["nodes"][0].pop("source_file"), + lambda doc: doc["nodes"][0].pop("file_type"), + lambda doc: doc["links"][0].pop("relation"), + lambda doc: doc["links"][0].pop("confidence"), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + with self.assertRaises(ContextError): + self.load(document) - def test_rejects_unknown_node_and_edge_kinds(self) -> None: + def test_refuses_vocabularies_the_providers_validator_rejects(self) -> None: for mutate in ( - lambda doc: doc["nodes"][0].update(kind="cluster"), - lambda doc: doc["edges"][0].update(kind="resembles"), - lambda doc: doc["edges"][0].update(evidence="guessed"), + lambda doc: doc["nodes"][0].update(file_type="diagram"), + lambda doc: doc["links"][0].update(confidence="GUESSED"), + # Lowercase is the packet contract's vocabulary, not the provider's. + lambda doc: doc["links"][0].update(confidence="extracted"), ): with self.subTest(mutate=mutate): document = graph_document() @@ -243,39 +386,29 @@ def test_rejects_unknown_node_and_edge_kinds(self) -> None: with self.assertRaises(ContextError): self.load(document) - def test_rejects_a_node_outside_the_indexed_checkout(self) -> None: + def test_refuses_an_unreadable_source_location(self) -> None: + for location in ("12", "line 12", "L", "L0", "L-4", "L99999999999"): + with self.subTest(location=location): + document = graph_document() + document["nodes"][0]["source_location"] = location + with self.assertRaises(ContextError): + self.load(document) + + def test_refuses_a_node_outside_the_indexed_checkout(self) -> None: """A node that could never be cited must not be traversable either.""" for path in ("/etc/passwd", "../sibling/config.py", ".git/config", ".graphify/nodes.bin"): with self.subTest(path=path): document = graph_document() - document["nodes"][0]["path"] = path + document["nodes"][0]["source_file"] = path with self.assertRaises(ContextError): self.load(document) - def test_rejects_a_dangling_edge(self) -> None: - document = graph_document() - document["edges"].append(edge("n-config", "n-missing", "calls")) - with self.assertRaises(ContextError): - self.load(document) - - def test_rejects_duplicate_node_identifiers(self) -> None: + def test_refuses_duplicate_node_identifiers(self) -> None: document = graph_document() document["nodes"].append(dict(document["nodes"][0])) with self.assertRaises(ContextError): self.load(document) - def test_rejects_an_inverted_line_span(self) -> None: - document = graph_document() - document["nodes"][0].update(start_line=30, end_line=12) - with self.assertRaises(ContextError): - self.load(document) - - def test_rejects_unrecognized_fields(self) -> None: - document = graph_document() - document["nodes"][0]["cluster"] = "semantic" - with self.assertRaises(ContextError): - self.load(document) - class TraversalTests(GraphWorkspace): def query(self, **overrides) -> query.QueryResult: @@ -332,7 +465,7 @@ def test_more_definitions_than_the_seed_bound_is_reported_as_truncation(self) -> document = graph_document() for index in range(query.MAX_SEEDS + 1): document["nodes"].append( - node(f"n-extra-{index}", "symbol", "parse_config", "example_pkg/loader.py", 1, 2)) + node(f"n-extra-{index}", "parse_config", "example_pkg/loader.py", 2)) graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) result = query.run_query(graph, question="impact", target="parse_config") self.assertEqual(len(result.seeds), query.MAX_SEEDS) @@ -368,7 +501,7 @@ def validator(self) -> query.CitationValidator: def test_validates_line_claims_against_the_bound_commit(self) -> None: validator = self.validator() - self.assertTrue(validator.validate("example_pkg/config.py#L12-L30")) + self.assertTrue(validator.validate("example_pkg/config.py#L12")) self.assertTrue(validator.validate("example_pkg/config.py#L40")) self.assertFalse(validator.validate("example_pkg/config.py#L41")) @@ -410,17 +543,17 @@ def test_every_citation_resolves_against_the_immutable_tree(self) -> None: def test_a_citation_past_the_end_of_a_file_is_dropped_not_delivered(self) -> None: """Evidence that cannot be pointed at is not weaker evidence; it is none.""" document = graph_document() - document["nodes"][1].update(start_line=400, end_line=440) + document["nodes"][1]["source_location"] = "L400" self.publish(document) outcome = self.context() cited = {citation["source"] for item in outcome.packet["documents"] for citation in item["citations"]} - self.assertNotIn("example_pkg/loader.py#L400-L440", cited) + self.assertNotIn("example_pkg/loader.py#L400", cited) self.assertIn("provider_warning", outcome.packet["omissions"]) def test_confidence_maps_extracted_inferred_and_ambiguous(self) -> None: document = graph_document() - document["edges"][2]["evidence"] = "ambiguous" + document["links"][2]["confidence"] = "AMBIGUOUS" self.publish(document) outcome = self.context() confidences = {item["confidence"] for item in outcome.packet["documents"]} @@ -450,7 +583,7 @@ def test_multi_hop_evidence_names_and_cites_the_edge_it_walked(self) -> None: self.assertIn("reached from parse_config", text) self.assertEqual( {citation["source"] for citation in documents[text]["citations"]}, - {"example_pkg/report.py#L5-L12", "example_pkg/loader.py#L40-L44"}, + {"example_pkg/report.py#L5", "example_pkg/loader.py#L40"}, ) # Each citation is titled with the node it points at, not with the node # the relationship happened to reach. @@ -525,8 +658,9 @@ def test_an_absent_graph_is_not_a_failure_of_the_caller(self) -> None: query.OPTIONAL_UNAVAILABLE) self.assertEqual(self.context().status, query.REQUIRED_UNAVAILABLE) - def test_a_generation_without_the_pinned_document_is_unreadable(self) -> None: - self.publish({"schema": "graphify.native.v1", "nodes": [], "edges": []}) + def test_a_generation_without_a_provider_export_is_unreadable(self) -> None: + """A member that is not the pinned exporter's document at all.""" + self.publish({"schema": "graphify.native.v1", "entities": [], "relations": []}) outcome = self.context() self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) self.assertEqual(outcome.summary["reason"], "unreadable") From 5de38f9e56da77916a1a692b2854e69cbdfb96e3 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:31:39 -0700 Subject: [PATCH 05/33] Graph queries: name the test-layout derivation's parts Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_graph_query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index db8d7d61..af5a72bc 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -156,7 +156,8 @@ #: A derivation, like ``_node_kind`` itself, and deliberately conservative: #: naming a non-test file a test would put it in a ``related_tests`` answer. _TEST_PREFIXES = ("tests/", "test/") -_TEST_STEMS = ("test_", "_test", ".test", "_spec", ".spec") +_TEST_DIRECTORIES = ("/tests/", "/test/") +_TEST_STEM_SUFFIXES = ("_test", "_spec") #: Relationship filters per question, and whether the traversal runs along #: edges or against them. ``impact`` asks who is affected by a change, which is @@ -367,10 +368,9 @@ def _node_kind(path: str, label: str, node_type: Any) -> str: stem = base.lower().rsplit(".", 1)[0] is_test = ( lowered.startswith(_TEST_PREFIXES) - or "/tests/" in lowered - or "/test/" in lowered + or any(directory in lowered for directory in _TEST_DIRECTORIES) or stem.startswith("test_") - or stem.endswith(("_test", "_spec")) + or stem.endswith(_TEST_STEM_SUFFIXES) ) if is_test: return "test" From c10bba9e0534725376fd1b084477dd839fb7ad37 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:44:27 -0700 Subject: [PATCH 06/33] Graph queries: reach the local graph from guided sessions The query module answered one bounded question and minted one packet, but only a standalone command could ask. This registers the same graph as an ordinary context connection so `session context prepare`, `deliver`, and the reuse and attachment that follow reach it through the shared packet store -- same protected handle, same authorization scope, same work item and recipient contract, same delivery path. A local connection has no principal, workspace, or credential. What replaces the credential is the graph itself: authorization is re-derived from current local state on every load and every replay, and the envelope carries the published generation as its `generation`. The freshness rules then fall out of the shared packet contract rather than out of new checks -- a rebuilt graph publishes a new generation and no longer matches a packet bound to the old one, and a moved HEAD makes the published generation stale for that revision so authorization fails outright. Required context that is refused pauses the dependent work; optional context degrades. `--question` is the retrieval source for this connection and the query names the symbol or path, both explicit: guessing a target out of a work item's prose would produce confident evidence about whatever happened to match. No Coworker SDK, credential, or network call takes part, and no recipient needs the provider, its pin, or any Graphify tool to read what it is given. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 52 ++- src/code_mower/context_delivery.py | 15 +- src/code_mower/context_graph_command.py | 38 ++- src/code_mower/context_graph_connection.py | 273 ++++++++++++++++ src/code_mower/context_packets.py | 83 +++-- src/code_mower/session.py | 16 +- tests/test_context_graph_connection.py | 358 +++++++++++++++++++++ 7 files changed, 805 insertions(+), 30 deletions(-) create mode 100644 src/code_mower/context_graph_connection.py create mode 100644 tests/test_context_graph_connection.py diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 1ac3234e..705cfdfb 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -237,11 +237,57 @@ is written to a freshly created private sibling and renamed over the destination, which is also atomic — a reader never sees a half-written packet, and a failed write leaves the previous file untouched. +## Guided sessions + +`context-graph query` is the standalone verb. The same graph is also reachable +from the ordinary guided route, by registering it as a context connection: + +``` +code-mower context-graph connect --connection local-graph \ + --repository owner/repo --recipient claude:builder --recipient codex:reviewer +code-mower context-graph connection-status --connection local-graph +code-mower context-graph disconnect --connection local-graph +``` + +A local connection has no principal, no workspace, and no credential. Nothing +is written to the OS credential vault, no browser opens, and no endpoint is +contacted; the connection's whole state is one checkout and the repositories +and recipients the operator approved for it. `session context prepare` then +reaches it through the shared packet store, with the same protected handle, the +same authorization scope, the same work item and recipient contract, and the +same attachment and delivery path an organization connection uses: + +``` +code-mower session context prepare SESSION.json \ + --question impact --query-stdin # stdin names the symbol or path +``` + +`--question` is this connection's retrieval source, and the query names the +symbol or repository-relative path. Both are explicit: a graph answers about a +named target, and guessing one out of a work item's prose would produce +confident evidence about whatever happened to match. `--question` defaults to +`symbol`. + +What replaces the credential is the graph. Authorization is re-derived from +current local state on every load and every replay — never cached — and the +envelope carries the **published generation** as its `generation`. Two rules +then fall out of the shared packet contract rather than out of new checks: + +- A **rebuilt** graph publishes a new generation, so a packet bound to the old + one no longer matches its envelope and is refused at load. +- A **moved `HEAD`** makes the published generation stale for that revision, so + authorization fails outright and nothing is delivered. + +Required context that is refused pauses the dependent work; optional context +degrades and the session continues with ordinary repository tools. Claude, +Codex and Devin receive byte-identical approved evidence, and no recipient +needs the provider, the pin, or any Graphify tool to read it. + ## Boundary This change adds no dependency, no background service, and no mandatory -indexing step. It does not wire the graph into `code-mower context fetch` or any -default guided-context selection: a graph packet is produced by an explicit -command, and the operator attaches it through the ordinary path. Hosted +indexing step. Nothing is selected by default: a graph is indexed only when an +operator builds one, and reached from a guided session only when an operator +connects one. Hosted Graphify, semantic or model-based extraction, clustering, watchers, and provider API keys remain out of scope and separate decisions. diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index fc8eab5b..a85e9efa 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -6,6 +6,7 @@ import uuid from dataclasses import dataclass, field +from . import context_graph_connection as graph_connection from . import context_review from .context_connections import _state from .context_contract import ContextError, ContextRequest, ValidatedPacket, _identifier, _object, _text, normalize_policy @@ -114,8 +115,18 @@ def reserve_attachment( "state": "available", "expires_at": payload["binding"]["expires_at"]}) render_evidence(packet, handle) with store.locked(name) as locked: - state = _state(locked.read(), name) - if state["state"] != "verified" or state["generation"] != payload["binding"]["generation"]: + saved = locked.read() + if graph_connection.is_graph(saved): + state = graph_connection.saved_state(saved, name) + # The local graph's "authorization changed" is a rebuild: the + # published generation is what a packet binds, so a graph rebuilt + # between preparation and attachment fails the same check a revoked + # organization authorization does. + generation = graph_connection.current_generation(state, root=store.root) + else: + state = _state(saved, name) + generation = state["generation"] + if state["state"] != "verified" or generation != payload["binding"]["generation"]: raise ContextError("context authorization changed before attachment") index_file, index = _index(locked) entry = next((item for item in index["entries"] if item["handle"] == handle), None) diff --git a/src/code_mower/context_graph_command.py b/src/code_mower/context_graph_command.py index a48595bb..58eb3ee4 100644 --- a/src/code_mower/context_graph_command.py +++ b/src/code_mower/context_graph_command.py @@ -20,10 +20,11 @@ import sys from pathlib import Path +from . import context_graph_connection as connection from . import context_graph_lifecycle as lifecycle from . import context_graph_query as query from .context_contract import ContextError -from .context_store import strict_json +from .context_store import ContextStore, strict_json MAX_PIN_BYTES = 8192 @@ -132,8 +133,15 @@ def main(argv=None) -> int: remove = sub.add_parser("remove", help="Delete this checkout's private local graph state") doctor = sub.add_parser("doctor", help="Check the local graph posture without building anything") ask = sub.add_parser("query", help="Answer one bounded question and emit a revision-bound packet") + # The guided route. ``query`` is the standalone verb an operator drives by + # hand; these three register the same graph as an ordinary context + # connection, so ``session context prepare``/``deliver`` can reach it + # through the shared packet store without a Coworker account or SDK. + join = sub.add_parser("connect", help="Register this checkout's graph as a local context connection") + leave = sub.add_parser("disconnect", help="Disable the connection and drop the packets it authorized") + linked = sub.add_parser("connection-status", help="Report the connection and the graph behind it") - for command in (build, refresh, status, remove, doctor, ask): + for command in (build, refresh, status, remove, doctor, ask, join, leave, linked): command.add_argument("--repo-path", type=Path, default=Path.cwd(), help="Checkout to bind") command.add_argument("--state-dir", type=Path, help="Private state root; defaults to the context store") command.add_argument("--json", action="store_true", help="Emit a machine-readable summary") @@ -157,6 +165,13 @@ def main(argv=None) -> int: ask.add_argument("--depth", type=int, help="Traversal depth; defaults to the question's own ceiling") ask.add_argument("--node-budget", type=int, default=query.DEFAULT_NODE_BUDGET, help="Most relationships one answer may carry before it reports truncation") + linked.add_argument("--revision", default="HEAD", help="Revision the connection must currently bind") + for command in (join, leave, linked): + command.add_argument("--connection", required=True, help="Context connection name to register or inspect") + join.add_argument("--repository", required=True, action="append", metavar="OWNER/NAME", + help="Approved context repository; repeat for more than one") + join.add_argument("--recipient", required=True, action="append", metavar="HOST:ROLE", + help="Approved recipient, for example claude:builder; repeat for more than one") args = parser.parse_args(argv) try: @@ -237,6 +252,25 @@ def main(argv=None) -> int: if key not in ("schema", "status")) _emit(outcome.summary, as_json=args.json, text="\n".join(lines) + "\n") return outcome.exit_code + if args.command in ("connect", "disconnect", "connection-status"): + store = ContextStore(args.state_dir) + if args.command == "connect": + summary = connection.connect(store, args.connection, { + "repository_root": str(Path(args.repo_path).resolve()), + "repositories": list(args.repository), + "recipients": list(args.recipient), + }) + elif args.command == "disconnect": + summary = connection.disconnect(store, args.connection) + else: + summary = connection.status( + store, args.connection, root=args.state_dir, revision=args.revision, + ) + lines = [f"Local graph connection: {summary['status']}"] + lines.extend(f" {key}: {value}" for key, value in sorted(summary.items()) + if key not in ("schema", "status")) + _emit(summary, as_json=args.json, text="\n".join(lines) + "\n") + return 0 if summary.get("authorization", "available") == "available" else 1 if args.command == "remove": state = lifecycle.GraphStateRoot(args.repo_path, root=args.state_dir) path = str(state.path) if args.show_local_paths else None diff --git a/src/code_mower/context_graph_connection.py b/src/code_mower/context_graph_connection.py new file mode 100644 index 00000000..9fafa8bc --- /dev/null +++ b/src/code_mower/context_graph_connection.py @@ -0,0 +1,273 @@ +"""Graphify as a local repository connection for the shared context store. + +The query module in ``context_graph_query`` answers one bounded question and +mints one packet. This module is what makes that answer reachable from the +ordinary guided route -- ``session context prepare``, ``deliver``, and the +reuse and attachment that follow -- without a second packet store, a second +delivery contract, or a second set of recipients. + +Three things are deliberately absent, because a local graph has none of them: +there is no principal, no workspace, and no credential. Connecting names a +checkout and the repositories and recipients an operator approves for it, and +that is the whole of the connection's state. Nothing here reads the OS +credential vault, opens a browser, or contacts a network endpoint. + +What replaces the credential is the graph itself. Authorization is not a saved +token that stays true until it expires; it is re-derived from current trusted +local state on every single load, from ``graph_status`` for the requested +revision, and the envelope it returns carries the *published generation* as its +``generation``. That one choice is what makes the freshness rules fall out of +the shared contract rather than out of new checks here: + +* A rebuilt graph publishes a new generation, so a packet minted against the + old one no longer matches the envelope, and ``load_packet`` refuses it. +* A moved ``HEAD`` makes ``graph_status`` report the published generation stale + for that revision, so authorization fails outright and nothing is delivered. + +A recipient therefore cannot be handed evidence from a graph that no longer +describes the code, and no recipient needs the provider, its pin, or any +Graphify tool to read what it is given. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping + +from . import context_graph_lifecycle as lifecycle +from . import context_graph_query as query +from .context_contract import ( + CAPABILITY_VERSION, CONNECTION_SCHEMA, ContextError, _identifier, _object, + _strings, _text, normalize_policy, validate_connection, +) +from .context_store import ContextStore + +#: Saved state for one local repository connection. Distinct from the private +#: Coworker connection schema so a store can hold both and every reader can +#: tell which contract it is looking at before it validates anything. +GRAPH_SCHEMA = "code_mower.contextLocalGraphConnection.v1" +PROVIDER = "graphify" +CONNECTION_KIND = "repository" + +#: Which bounded question a retrieval asks when the caller names none. The +#: guided route carries a free-text ``source`` for exactly this purpose. +DEFAULT_QUESTION = "symbol" + +#: How long one live authorization stands. Short on purpose: it bounds a packet +#: that was already minted, and the graph behind it is re-checked on every load +#: regardless, so a longer window would buy nothing and hide a stale build for +#: longer if a check were ever skipped. +AUTHORIZATION_SECONDS = 3600 + + +def is_graph(value: Any) -> bool: + """Whether saved connection state belongs to this contract. + + Cheap and structural, so a caller can branch before validating. A state + that claims this schema and then fails ``saved_state`` is an error, not a + reason to fall through to the organization path. + """ + return isinstance(value, Mapping) and value.get("schema") == GRAPH_SCHEMA + + +def connection_spec(value: Any) -> dict[str, Any]: + """The operator's approval: one checkout, and who may read answers from it.""" + spec = _object(value, {"repository_root", "repositories", "recipients"}) + root = Path(_text(spec["repository_root"], maximum=4096)) + if not root.is_absolute(): + raise ContextError("local context repository root must be absolute") + return { + "repository_root": str(root), + "repositories": list(_strings(spec["repositories"])), + "recipients": list(_strings(spec["recipients"])), + } + + +def saved_state(value: Any, name: str) -> dict[str, Any]: + if value is None: + raise ContextError("context connection is missing; run context-graph connect") + state = _object(value, {"schema", "connection", "provider", "kind", "state", + "repository_root", "repositories", "recipients"}) + if (state["schema"] != GRAPH_SCHEMA or state["connection"] != name + or state["provider"] != PROVIDER or state["kind"] != CONNECTION_KIND + or state["state"] not in {"verified", "disconnected"}): + raise ContextError("unsupported local graph context state; reconnect") + return {**state, **connection_spec({key: state[key] for key in + ("repository_root", "repositories", "recipients")})} + + +def _summary(state: Mapping[str, Any]) -> dict[str, Any]: + return {"schema": "code_mower.contextConnectionSummary.v1", "provider": PROVIDER, + "kind": CONNECTION_KIND, "status": state["state"], "search": "available", + "memory": "unavailable", "credential_storage": "none"} + + +def connect(store: ContextStore, name: str, spec: Any) -> dict[str, Any]: + """Record one approved checkout. No credential is created or stored.""" + name = _identifier(name) + spec = connection_spec(spec) + # Resolving now rather than at every load: the identity a packet binds is + # the one the operator approved, and a root that later becomes a symlink + # elsewhere must not quietly redirect the evidence. + root = lifecycle.checkout_root(Path(spec["repository_root"])) + with store.locked(name, timeout_seconds=1) as locked: + old = locked.read() + if old is not None and not is_graph(old): + raise ContextError("this connection name already names a different context provider") + if old is not None and saved_state(old, name)["state"] != "disconnected": + raise ContextError("connection already exists; disconnect before changing its checkout or scope") + state = {"schema": GRAPH_SCHEMA, "connection": name, "provider": PROVIDER, + "kind": CONNECTION_KIND, "state": "verified", "repository_root": str(root), + "repositories": spec["repositories"], "recipients": spec["recipients"]} + locked.write(state) + return _summary(state) + + +def disconnect(store: ContextStore, name: str) -> dict[str, Any]: + """Disable the connection first, then drop the packets it authorized.""" + with store.locked(name) as locked: + state = {**saved_state(locked.read(), name), "state": "disconnected"} + locked.write(state) + packet_cleanup = "complete" + try: + from .context_packets import purge_connection + purge_connection(locked) + except Exception: + packet_cleanup = "needs_attention" + return {**_summary(state), "packet_cleanup": packet_cleanup} + + +def status(store: ContextStore, name: str, *, root: Path | None = None, + revision: str = "HEAD") -> dict[str, Any]: + """Report the connection and the graph behind it, without minting evidence.""" + with store.locked(name) as locked: + state = saved_state(locked.read(), name) + report = lifecycle.graph_status(Path(state["repository_root"]), root=root, revision=revision) + return {**_summary(state), "graph": report.shareable_summary(), + "authorization": "available" if state["state"] == "verified" and report.usable + else "unavailable"} + + +def _published(state: Mapping[str, Any], *, root: Path | None, + revision: str) -> lifecycle.GenerationStatus: + if state["state"] != "verified": + raise ContextError("local graph context is disconnected; reconnect before using it") + report = lifecycle.graph_status(Path(state["repository_root"]), root=root, revision=revision) + if not report.usable or report.manifest is None: + # The state word is the lifecycle's own -- ``stale``, ``partial``, + # ``absent`` -- and it is the only detail worth carrying: it says + # whether to rebuild, refresh, or build for the first time. + raise ContextError( + "local graph is not current for this revision (" + report.state + "); rebuild it" + ) + return report + + +def current_generation(state: Mapping[str, Any], *, root: Path | None = None, + revision: str = "HEAD") -> str: + """The published generation a packet must still match to be usable.""" + manifest = _published(state, root=root, revision=revision).manifest + assert manifest is not None # a usable status always carries one + return manifest.generation + + +def authorize_locked(locked, name: str, *, root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> dict[str, Any]: + """Mint one envelope from current local state, under the caller's lock. + + This is the local counterpart of the organization connection's online + refresh, and it is deliberately the same shape: every load and every replay + calls it again, and none of them may reuse a previous answer. What it + checks is not a token but the graph -- present, complete, and binding the + requested revision -- and what it publishes as ``generation`` is the + graph's, so the shared packet contract does the rest. + """ + current = now or datetime.now(timezone.utc) + state = saved_state(locked.read(), name) + manifest = _published(state, root=root, revision=revision).manifest + assert manifest is not None + envelope = { + "schema": CONNECTION_SCHEMA, "capability_version": CAPABILITY_VERSION, + "connection": name, "provider": PROVIDER, "kind": CONNECTION_KIND, + "generation": manifest.generation, "state": "verified", + "identity": {"repository_root": state["repository_root"]}, + "repositories": list(state["repositories"]), "recipients": list(state["recipients"]), + "expires_at": (current + timedelta(seconds=AUTHORIZATION_SECONDS)).isoformat(), + # No memory, and a real revision binding: the packet names the commit + # the graph was built from, so a recipient asking about another one + # resolves ``stale`` without decoding the evidence. + "capabilities": {"search": True, "memory": False, "revision_binding": True}, + } + return dict(validate_connection(envelope, now=current)) + + +def question_and_target(spec: Mapping[str, Any]) -> tuple[str, str]: + """Read one bounded question out of the shared retrieval request. + + The guided request carries a free-text ``query`` and an optional ``source``. + For a graph those are the target and the question, and both are explicit on + purpose: this connection answers about a named symbol or a repository- + relative path, and guessing one out of a work item's prose would produce + confident evidence about whatever happened to match. + """ + question = spec.get("source") or DEFAULT_QUESTION + if question not in query.QUESTIONS: + raise ContextError( + "local graph source must name a question: " + ", ".join(query.QUESTIONS) + ) + target = _text(spec["query"], maximum=2000) + return question, target + + +def retrieve(state: Mapping[str, Any], spec: Mapping[str, Any], *, envelope: Mapping[str, Any], + root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> dict[str, Any]: + """Answer once from the published generation, or say why there is no answer. + + ``graph_context`` returns rather than raises when the graph cannot answer, + because "no graph" is a normal state of an opt-in feature. The packet store + is not a place where that distinction survives -- a reservation either + produces a packet or it does not -- so the unavailable outcomes become the + error the store already knows how to unwind, carrying the reason forward. + """ + question, target = question_and_target(spec) + policy = normalize_policy(spec["policy"]) + if policy is None: + raise ContextError("context policy is not configured") + outcome = query.graph_context( + Path(state["repository_root"]), + question=question, + target=target, + envelope=envelope, + policy=policy, + context_repository=spec["repository"], + work_item=spec["work_item"], + root=root, + revision=revision, + now=now, + ) + if outcome.packet is None: + raise ContextError( + "local graph context is unavailable (" + str(outcome.summary.get("reason", "unknown")) + ")" + ) + return outcome.packet + + +__all__ = ( + "AUTHORIZATION_SECONDS", + "CONNECTION_KIND", + "DEFAULT_QUESTION", + "GRAPH_SCHEMA", + "PROVIDER", + "authorize_locked", + "connect", + "connection_spec", + "current_generation", + "disconnect", + "is_graph", + "question_and_target", + "retrieve", + "saved_state", + "status", +) diff --git a/src/code_mower/context_packets.py b/src/code_mower/context_packets.py index de9a6b1a..f4d3c6ee 100644 --- a/src/code_mower/context_packets.py +++ b/src/code_mower/context_packets.py @@ -11,6 +11,7 @@ from datetime import datetime, timedelta, timezone from pathlib import Path +from . import context_graph_connection as graph_connection from .context_connections import _backend, _state, authorize_locked from .context_contract import ( CAPABILITY_VERSION, PACKET_SCHEMA, ContextError, ContextRequest, _object, @@ -114,17 +115,33 @@ def _load(store, entry, policy, request, envelope): request=request, authorize=lambda: envelope) -def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): - """Retrieve once, or reauthorize and reuse; never redispatch automatically.""" +def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revision="HEAD"): + """Retrieve once, or reauthorize and reuse; never redispatch automatically. + + Which provider answers is the connection's own saved state, read here under + the same lock that guards the retrieval. A local repository graph reaches + the same index, the same packet files, and the same delivery contract as an + organization connection; what differs is only where authorization and + evidence come from, and neither kind can be mistaken for the other because + the saved schema is checked before either path is taken. + """ spec = request_spec(spec, name) policy = spec["policy"] started = time.monotonic() - backend = backend or _backend() with store.locked(name, timeout_seconds=policy["timeout_seconds"]) as locked: + local = graph_connection.is_graph(locked.read()) left = policy["timeout_seconds"] - (time.monotonic() - started) if left <= 0: raise ContextError("context retrieval deadline exceeded before authorization") - envelope = authorize_locked(locked, name, backend, timeout_seconds=min(left, 30)) + if local: + envelope = graph_connection.authorize_locked( + locked, name, root=store.root, revision=revision, + ) + else: + # Deferred until the connection kind is known: a local graph must + # not require the optional provider SDK to be installed at all. + backend = backend or _backend() + envelope = authorize_locked(locked, name, backend, timeout_seconds=min(left, 30)) if spec["repository"] not in envelope["repositories"] or spec["recipient"] not in envelope["recipients"]: raise ContextError("context connection does not authorize this repository or recipient") fingerprint = _key({k: v for k, v in spec.items() if k != "recipient"}) @@ -149,42 +166,66 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False): left = policy["timeout_seconds"] - (time.monotonic() - started) if left <= 0: raise ContextError("context retrieval deadline exceeded; no search was sent") - state = _state(locked.read(), name) - credentials = locked.vault.get(state["credential_id"]) + state = graph_connection.saved_state(locked.read(), name) if local else _state(locked.read(), name) try: - result = backend.retrieve(credentials, spec["query"], spec["source"], policy, timeout_seconds=left) - now = datetime.now(timezone.utc) - expiry = min(_timestamp(envelope["expires_at"]), now + timedelta(seconds=policy["max_age_seconds"])) - packet_data = {"schema": PACKET_SCHEMA, "capability_version": CAPABILITY_VERSION, - "provider": "coworker", "kind": "organization", "retrieved_at": now.isoformat(), - **{key: result[key] for key in ("documents", "completeness", "truncated", "source_revision", "source_built_at", "omissions")}, - "binding": {**{key: envelope[key] for key in ("connection", "generation", "identity", "recipients")}, - "repository": spec["repository"], "work_item": spec["work_item"], - "policy_version": policy["policy_version"], "expires_at": expiry.isoformat()}} + if local: + # The local graph mints its own packet: it decides what the + # evidence is, and the envelope above decides who may read it. + packet_data = graph_connection.retrieve( + state, spec, envelope=envelope, root=store.root, revision=revision, + ) + usage = None + else: + credentials = locked.vault.get(state["credential_id"]) + result = backend.retrieve(credentials, spec["query"], spec["source"], policy, timeout_seconds=left) + now = datetime.now(timezone.utc) + expiry = min(_timestamp(envelope["expires_at"]), now + timedelta(seconds=policy["max_age_seconds"])) + packet_data = {"schema": PACKET_SCHEMA, "capability_version": CAPABILITY_VERSION, + "provider": "coworker", "kind": "organization", "retrieved_at": now.isoformat(), + **{key: result[key] for key in ("documents", "completeness", "truncated", "source_revision", "source_built_at", "omissions")}, + "binding": {**{key: envelope[key] for key in ("connection", "generation", "identity", "recipients")}, + "repository": spec["repository"], "work_item": spec["work_item"], + "policy_version": policy["policy_version"], "expires_at": expiry.isoformat()}} + usage = result["usage"] locked.artifact("p-" + entry["handle"]).write(packet_data) raw = json.dumps(packet_data, allow_nan=False, separators=(",", ":")).encode() entry["reference"] = {"path": ".p-" + entry["handle"] + ".json", "sha256": hashlib.sha256(raw).hexdigest()} packet = _load(store, entry, policy, _request(spec), envelope) - entry["usage"] = result["usage"] + entry["usage"] = usage index_file.write(index) _index(locked) - locked.write({**state, "capability_status": {"search": "available", "memory": "available"}}) + if not local: + locked.write({**state, "capability_status": {"search": "available", "memory": "available"}}) except Exception: entry["reference"] = None entry["usage"] = None index_file.write(index) locked.artifact("p-" + entry["handle"]).delete() - locked.write({**state, "capability_status": {"search": "unavailable", "memory": "unavailable"}}) + if not local: + locked.write({**state, "capability_status": {"search": "unavailable", "memory": "unavailable"}}) raise ContextError("context search unavailable; no automatic retry; verify access or explicitly refresh") from None return {**packet.shareable_summary(), "status": "available", "packet_handle": entry["handle"], "reused": False, "usage": entry["usage"]} -def load_authorized(store, name, handle, policy, request: ContextRequest, *, backend=None): - """Every participant replay obtains a new online authorization under lock.""" +def load_authorized(store, name, handle, policy, request: ContextRequest, *, backend=None, + revision="HEAD"): + """Every participant replay obtains a new authorization under lock. + + For an organization connection that is a fresh online check. For a local + repository graph it is a fresh read of current local state: the published + generation for ``revision``. Either way the envelope is minted here and + now, so a packet whose graph was rebuilt or whose revision has moved on is + refused by the shared contract rather than replayed. + """ _handle(handle) with store.locked(name) as locked: - envelope = authorize_locked(locked, name, backend or _backend()) + if graph_connection.is_graph(locked.read()): + envelope = graph_connection.authorize_locked( + locked, name, root=store.root, revision=revision, + ) + else: + envelope = authorize_locked(locked, name, backend or _backend()) _file, index = _index(locked) entry = next((entry for entry in index["entries"] if entry["handle"] == handle), None) if entry is None: diff --git a/src/code_mower/session.py b/src/code_mower/session.py index b9cf3639..94573022 100644 --- a/src/code_mower/session.py +++ b/src/code_mower/session.py @@ -12,7 +12,10 @@ from pathlib import Path from typing import Any, Mapping -from . import context_guided, context_prepare, context_session, remote_session_cli, session_current, session_lease +from . import ( + context_graph_query, context_guided, context_prepare, context_session, + remote_session_cli, session_current, session_lease, +) from .config import ConfigError, _format_issues, load_config, validate_config from .context_contract import ContextError, normalize_policy from .context_store import ContextStore @@ -376,7 +379,12 @@ def _run_context_command(args: argparse.Namespace) -> tuple[dict[str, Any], int] ) return {"private_text": text}, 0 tracker = source.get("tracker") - retrieval_source = ( + # ``--question`` is the local repository graph's retrieval source: that + # connection answers one bounded question about a named symbol or path, and + # the question is the operator's to choose. It takes precedence over the + # tracker-derived source, which only means anything to an organization + # search; a checkout cannot be connected to both at one connection name. + retrieval_source = getattr(args, "question", None) or ( "jira" if isinstance(tracker, Mapping) and tracker.get("kind") == "jira_cloud" else None ) return context_prepare.prepare( @@ -478,6 +486,10 @@ def main(argv: list[str] | None = None) -> int: "--query-stdin", action="store_true", help="read a private query override from stdin; the selected work item is the default", ) + context_prepare_parser.add_argument( + "--question", choices=context_graph_query.QUESTIONS, + help="bounded question for a local repository graph connection; the query names the symbol or path", + ) context_prepare_parser.add_argument("--work-order-body-file", type=Path) context_prepare_parser.add_argument("--title") context_prepare_parser.add_argument( diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py new file mode 100644 index 00000000..99ca236d --- /dev/null +++ b/tests/test_context_graph_connection.py @@ -0,0 +1,358 @@ +"""The guided route over a local repository graph (issue #914). + +These tests drive the ordinary guided path -- ``context_prepare.prepare``, the +shared packet store, and ``context_delivery`` -- against a Graphify-kind local +connection rather than an organization one. What they are here to prove is not +the packet format, which ``test_context_graph_query`` already covers, but that +the guided route reaches it: that preparation mints a packet through the shared +store, that reuse does not re-query, that Claude, Codex and Devin receive +byte-identical approved evidence, and that a graph which was rebuilt or whose +revision has moved on is refused rather than replayed. + +No Coworker SDK, credential, or network call takes part. The graph document is +written by an injected indexer, as in the query tests, so nothing here claims a +provider was installed. +""" + +from __future__ import annotations + +import os +import tempfile +import unittest +from datetime import datetime, timezone +from pathlib import Path + +from code_mower import context_delivery, context_packets, context_prepare, context_session +from code_mower import context_graph_connection as connection +from code_mower import context_graph_lifecycle as lifecycle +from code_mower import context_graph_query as query +from code_mower.context_contract import ContextError, ContextRequest +from code_mower.context_store import ContextStore +from test_context_connections import MemoryVault +from test_context_graph_query import PIN, git, graph_document, indexer, make_repository + + +POLICY = { + "schema": "code_mower.contextPolicy.v1", + "connection": "local-graph", + "policy_version": "v1", + "required": True, +} +RECIPIENTS = [ + "codex:orchestrator", "claude:orchestrator", "devin:orchestrator", + "codex:builder", "claude:builder", "devin:builder", + "codex:reviewer", "claude:reviewer", "devin:reviewer", +] + + +def session_value(session_id: str = "a" * 32, *, host: str = "codex") -> dict: + return { + "schema": "code_mower.session.v1", + "id": session_id, + "repo": "owner/repo", + "host": host, + "orchestrator": host, + "participants": [{"id": "claude"}, {"id": "codex"}], + "lease": {"state": "held", "mutating": True}, + } + + +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class GuidedGraphSessionTests(unittest.TestCase): + """One checkout, one published generation, one guided session over it.""" + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = make_repository(self.root) + # One private root for both the graph's generations and the packet + # store, which is the arrangement an operator actually gets: the + # lifecycle and the context store share ``default_context_root``. + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = self.publish() + self.store = ContextStore(self.private, vault=MemoryVault()) + self.associations = context_session.association_store(self.private) + connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + }) + + def publish(self, document: dict | None = None): + return lifecycle.build_graph( + self.repository, + pin=PIN, + indexer=indexer(document or graph_document()), + root=self.private, + ) + + def spec(self, *, question: str = "impact", target: str = "parse_config", + recipient: str = "codex:orchestrator", required: bool = True) -> dict: + return { + "repository": "owner/repo", "work_item": "WORK-1", "recipient": recipient, + "query": target, "source": question, + "policy": {**POLICY, "required": required}, + } + + def fetch(self, **overrides): + return context_packets.fetch(self.store, "local-graph", self.spec(**overrides)) + + def load(self, handle: str, recipient: str): + return context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", recipient), + ) + + # -- retrieval through the shared store ------------------------------ + + def test_retrieval_binds_the_published_generation_and_the_graphs_commit(self) -> None: + result = self.fetch() + self.assertEqual((result["status"], result["reused"]), ("available", False)) + # No paid-provider usage exists for a local graph, and reporting a + # fabricated zero would read as "a search happened and cost nothing". + self.assertIsNone(result["usage"]) + packet = self.load(result["packet_handle"], "claude:builder").private_payload() + self.assertEqual(packet["provider"], connection.PROVIDER) + self.assertEqual(packet["kind"], "repository") + self.assertEqual(packet["source_revision"], self.manifest.commit) + self.assertEqual(packet["binding"]["generation"], self.manifest.generation) + self.assertEqual( + packet["binding"]["identity"], {"repository_root": str(self.repository)}, + ) + + def test_reuse_returns_the_same_packet_without_traversing_again(self) -> None: + first = self.fetch() + again = self.fetch() + self.assertEqual(again["packet_handle"], first["packet_handle"]) + self.assertTrue(again["reused"]) + + def test_a_different_question_is_a_different_packet(self) -> None: + impact = self.fetch(question="impact") + symbol = self.fetch(question="symbol") + self.assertNotEqual(symbol["packet_handle"], impact["packet_handle"]) + self.assertFalse(symbol["reused"]) + + def test_an_unsupported_question_is_refused_before_any_traversal(self) -> None: + with self.assertRaises(ContextError): + self.fetch(question="everything") + + def test_an_unapproved_recipient_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.fetch(recipient="cursor:builder") + + # -- identical approved evidence ------------------------------------- + + def test_claude_codex_and_devin_receive_identical_approved_evidence(self) -> None: + handle = self.fetch()["packet_handle"] + rendered = { + host: context_delivery.render_evidence( + self.load(handle, f"{host}:builder"), handle, + ) + for host in context_delivery.SUPPORTED_HOSTS + } + self.assertEqual(set(rendered), {"claude", "codex", "devin"}) + self.assertEqual(len(set(rendered.values())), 1) + evidence = rendered["claude"] + self.assertIn("example_pkg/config.py#L12", evidence) + # The recipient needs no provider, pin, or graph tool to read this: the + # evidence names the provider, as it does for an organization packet, + # and carries no private path, checkout root, or connection identity. + for private in (str(self.private), str(self.repository), "local-graph"): + self.assertNotIn(private, evidence) + + # -- freshness: rebuilt, moved, or absent ---------------------------- + + def test_a_rebuilt_graph_refuses_the_packet_bound_to_the_old_generation(self) -> None: + handle = self.fetch()["packet_handle"] + republished = self.publish() + self.assertNotEqual(republished.generation, self.manifest.generation) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_a_moved_head_refuses_delivery_rather_than_answering_for_the_old_commit(self) -> None: + handle = self.fetch()["packet_handle"] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_an_explicitly_named_prior_revision_still_loads_its_own_packet(self) -> None: + """Freshness is about the requested revision, not about wall-clock time.""" + handle = self.fetch()["packet_handle"] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "second") + packet = context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder"), + revision=self.manifest.commit, + ) + self.assertEqual(packet.private_payload()["source_revision"], self.manifest.commit) + + def test_removing_the_graph_makes_the_connection_unavailable(self) -> None: + handle = self.fetch()["packet_handle"] + lifecycle.remove_graph(self.repository, root=self.private) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_disconnecting_disables_the_connection_and_drops_its_packets(self) -> None: + handle = self.fetch()["packet_handle"] + summary = connection.disconnect(self.store, "local-graph") + self.assertEqual((summary["status"], summary["packet_cleanup"]), ("disconnected", "complete")) + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + + def test_connection_status_reports_the_graph_without_minting_evidence(self) -> None: + report = connection.status(self.store, "local-graph", root=self.private) + self.assertEqual(report["provider"], connection.PROVIDER) + self.assertEqual(report["authorization"], "available") + self.assertTrue(report["graph"]["usable"]) + + def test_authorization_names_a_repository_kind_without_an_account(self) -> None: + with self.store.locked("local-graph") as locked: + envelope = connection.authorize_locked( + locked, "local-graph", root=self.private, now=datetime.now(timezone.utc), + ) + self.assertEqual(envelope["kind"], "repository") + self.assertEqual(envelope["generation"], self.manifest.generation) + self.assertEqual( + envelope["capabilities"], + {"search": True, "memory": False, "revision_binding": True}, + ) + self.assertNotIn("principal", envelope["identity"]) + + # -- the guided session verbs ---------------------------------------- + + def record(self, *, host: str = "codex", required: bool = True): + return context_session.create( + self.associations, + session_value(host=host), + work_item="WORK-1", + policy={**POLICY, "required": required}, + ) + + def prepare(self, record, **kwargs): + arguments = {"query": "parse_config", "source": "impact"} + arguments.update(kwargs) + return context_prepare.prepare( + self.associations, + record, + repo_root=self.repository, + context_root=self.private, + packet_store=self.store, + **arguments, + ) + + def test_prepare_then_reuse_produces_one_packet_the_builder_can_read(self) -> None: + record = self.record() + first, code = self.prepare(record) + self.assertEqual((code, first["status"], first["reused"]), (0, "prepared", False)) + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual(saved["stage"], "prepared") + self.assertIsNotNone(saved["packet"]) + + again, code = self.prepare(saved) + self.assertEqual((code, again["status"], again["reused"]), (0, "prepared", True)) + self.assertEqual( + context_session.read(self.associations, record["session_id"])["packet"], + saved["packet"], + ) + evidence = context_delivery.render_evidence( + self.load(saved["packet"], "codex:builder"), saved["packet"], + ) + self.assertIn("example_pkg/config.py#L12", evidence) + + def test_prepare_pauses_required_work_when_the_graph_is_stale(self) -> None: + record = self.record(required=True) + lifecycle.remove_graph(self.repository, root=self.private) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + self.assertEqual(report["dependent_work"], "paused") + + def test_prepare_degrades_optional_work_when_the_graph_is_stale(self) -> None: + record = self.record(required=False) + lifecycle.remove_graph(self.repository, root=self.private) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + self.assertEqual(report["dependent_work"], "usable") + + def test_reuse_after_a_rebuild_pauses_required_work(self) -> None: + record = self.record(required=True) + self.prepare(record) + saved = context_session.read(self.associations, record["session_id"]) + self.publish() + report, code = self.prepare(saved) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + + def test_attachment_refuses_a_packet_whose_graph_was_rebuilt(self) -> None: + handle = self.fetch()["packet_handle"] + self.publish() + with self.assertRaises(ContextError): + context_delivery.reserve_attachment( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "codex:orchestrator"), + pr=1, head="c" * 40, + ) + + +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class GraphConnectionStateTests(unittest.TestCase): + """What the saved connection will and will not accept.""" + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.repository = make_repository(self.root) + self.store = ContextStore(self.private, vault=MemoryVault()) + + def connect(self, **overrides): + spec = { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": ["claude:builder"], + } + spec.update(overrides) + return connection.connect(self.store, "local-graph", spec) + + def test_connecting_stores_no_credential_and_reports_no_vault(self) -> None: + summary = self.connect() + self.assertEqual(summary["credential_storage"], "none") + self.assertEqual(summary["kind"], "repository") + + def test_a_relative_checkout_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.connect(repository_root="checkout") + + def test_reconnecting_a_live_connection_is_refused(self) -> None: + self.connect() + with self.assertRaises(ContextError): + self.connect(repositories=["owner/other"]) + + def test_reconnecting_after_disconnect_is_allowed(self) -> None: + self.connect() + connection.disconnect(self.store, "local-graph") + self.assertEqual(self.connect(repositories=["owner/other"])["status"], "verified") + + def test_the_question_defaults_to_symbol_and_rejects_anything_else(self) -> None: + self.assertEqual( + connection.question_and_target({"query": "parse_config", "source": None}), + (connection.DEFAULT_QUESTION, "parse_config"), + ) + self.assertIn(connection.DEFAULT_QUESTION, query.QUESTIONS) + with self.assertRaises(ContextError): + connection.question_and_target({"query": "parse_config", "source": "anything"}) + + def test_saved_state_refuses_another_providers_connection(self) -> None: + self.assertFalse(connection.is_graph({"schema": "code_mower.contextLocalConnection.v1"})) + with self.assertRaises(ContextError): + connection.saved_state({"schema": connection.GRAPH_SCHEMA}, "local-graph") + + +if __name__ == "__main__": # pragma: no cover - direct invocation + unittest.main() From 7ec69d5bf87eb89bc65d484aa21304193bb687a6 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 00:56:32 -0700 Subject: [PATCH 07/33] Graph queries: bind the generation the evidence came from Three repairs from the audit at `c10bba9`, all about a claim the code made without checking it. The packet's binding copied the authorization envelope's `generation` straight through. Every freshness rule in the delivery contract reads that field -- a rebuilt graph is refused precisely because the published generation moved -- so a packet bound to a generation its traversal never read would go on passing all of them. `build_packet` now refuses when the authorized generation and the queried one disagree. It does not rewrite the envelope to match: that is an authorization this module did not mint, and the disagreement is the answer. The guided route is unaffected, since `authorize_locked` and `graph_context` both read current local state; the standalone command path is where an envelope from another moment could arrive. The query tests now authorize against what is published at call time, which is what the connection actually does, rather than a literal fixed at setUp. Both provider vocabulary fields were tested for membership before being tested for type. `file_type` is looked up in a set and `confidence` in a dict, so a JSON array or object in either raised TypeError out of a reader whose callers only catch ContextError -- an opt-in feature crashing instead of reporting the graph unreadable. Both are read as bounded text first, and the graph-context outcomes are covered both ways: required blocks, optional degrades. `context_graph_connection.py` was missing from the package manifest. `context_packets` and `context_delivery` import it, so an installed package would have lost the context routes entirely. Co-Authored-By: Claude Opus 5 (1M context) --- code-mower-package-manifest.json | 5 ++ src/code_mower/context_graph_query.py | 24 +++++++-- src/code_mower/package_manifest.py | 5 ++ tests/test_context_graph_query.py | 73 +++++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/code-mower-package-manifest.json b/code-mower-package-manifest.json index b5fdb0c7..8fb884f9 100644 --- a/code-mower-package-manifest.json +++ b/code-mower-package-manifest.json @@ -707,6 +707,11 @@ "source": "src/code_mower/context_graph_command.py", "target": "src/code_mower/context_graph_command.py" }, + { + "kind": "core", + "source": "src/code_mower/context_graph_connection.py", + "target": "src/code_mower/context_graph_connection.py" + }, { "kind": "core", "source": "src/code_mower/context_graph_lifecycle.py", diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index af5a72bc..82fa0513 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -390,7 +390,10 @@ def _node(value: Any) -> GraphNode | None: ``load_graph`` prunes and counts. """ record = _required(value, GRAPH_NODE_FIELDS, what="node") - file_type = record["file_type"] + # Bounded text before membership: a vocabulary field is looked up in a set, + # and a JSON array or object there is unhashable, so testing it first + # raises TypeError out of a reader whose only failure is ``ContextError``. + file_type = _text(record["file_type"], maximum=64) if file_type not in GRAPH_FILE_TYPES: raise ContextError("unsupported local graph node file type") if file_type != CODE_FILE_TYPE: @@ -427,7 +430,10 @@ def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge | None: answer instead of present with one end unstated. """ record = _required(value, GRAPH_EDGE_FIELDS, what="edge") - confidence = record["confidence"] + # Bounded text first, for the same reason as a node's ``file_type``: the + # lookup below is a dict membership test, which a non-text value turns into + # a TypeError instead of the refusal this module promises. + confidence = _text(record["confidence"], maximum=64) if confidence not in GRAPH_CONFIDENCES: raise ContextError("unsupported local graph edge confidence") relation = _text(record["relation"], maximum=128) @@ -912,7 +918,9 @@ def build_packet( generation as its provenance, so a consumer that asked about a different revision resolves ``stale`` at delivery without decoding the payload. The binding is the authorization envelope's, unchanged: this module decides - what the evidence is, never who may read it. + what the evidence is, never who may read it. What it does check is that the + two describe one graph -- an envelope authorizing a generation the + traversal did not read is refused rather than reconciled. """ current = now or datetime.now(timezone.utc) if current.tzinfo is None: @@ -923,6 +931,16 @@ def build_packet( connection = validate_connection(envelope, now=current) if connection["kind"] != "repository": raise ContextError("local graph evidence requires a repository-kind connection") + # The binding names one generation, and the freshness rules the delivery + # contract enforces all read it: a rebuilt graph is refused because the + # published generation moved. Copying the envelope's word for it would make + # that check vacuous whenever the traversal came from a different + # generation than the one authorized -- the packet would claim provenance + # it does not have, and still pass every later comparison. The envelope is + # not rewritten to match, because it is an authorization this module did + # not mint: the disagreement is the refusal. + if connection["generation"] != result.generation: + raise ContextError("local graph evidence is not from the authorized graph generation") if completeness not in (lifecycle.COMPLETE, lifecycle.PARTIAL): raise ContextError("unsupported local graph completeness") if not result.resolved: diff --git a/src/code_mower/package_manifest.py b/src/code_mower/package_manifest.py index aa93aaa0..c624cf5d 100644 --- a/src/code_mower/package_manifest.py +++ b/src/code_mower/package_manifest.py @@ -52,6 +52,11 @@ "src/code_mower/context_graph_command.py", "core", ), + ( + "src/code_mower/context_graph_connection.py", + "src/code_mower/context_graph_connection.py", + "core", + ), ("src/code_mower/productivity_report.py", "src/code_mower/productivity_report.py", "core"), ("tools/code_mower_requirements.txt", "requirements/requirements.txt", "tooling"), ("tools/code_mower_calibration.py", "src/code_mower/code_mower_calibration.py", "core"), diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 16ee2165..e96dec1f 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -229,11 +229,23 @@ def publish(self, document: dict, *, completeness: str | None = None): now=NOW, ) + def authorized(self, **overrides) -> dict: + """An envelope as the connection mints one: bound to what is published now. + + ``authorize_locked`` reads the current generation on every call rather + than remembering one, and a packet may only carry the generation its + evidence came from. A literal fixed at ``setUp`` would authorize one + generation and cite another as soon as a test rebuilds the graph -- + the disagreement the packet builder refuses. + """ + published = lifecycle.graph_status(self.repository, root=self.state).generation + return envelope(**{"generation": published, **overrides}) + def context(self, **overrides) -> query.GraphContext: arguments = { "question": "impact", "target": "parse_config", - "envelope": envelope(), + "envelope": self.authorized(), "policy": policy(), "context_repository": "owner/repo", "work_item": "work-item-one", @@ -258,7 +270,7 @@ def load(self, packet: dict, *, recipient: str, revision: str | None): reference={"path": "delivered-packet.json", "sha256": hashlib.sha256(encoded).hexdigest()}, policy=policy(), request=contract.ContextRequest("owner/repo", "work-item-one", recipient, revision), - authorize=lambda: envelope(), + authorize=lambda: self.authorized(), now=NOW, ) @@ -386,6 +398,22 @@ def test_refuses_vocabularies_the_providers_validator_rejects(self) -> None: with self.assertRaises(ContextError): self.load(document) + def test_refuses_a_vocabulary_field_that_is_not_text_at_all(self) -> None: + """A JSON array or object where a word belongs is a refusal, not a crash. + + Both vocabulary fields are checked by membership in a set or a dict, and + an unhashable value there raises ``TypeError`` -- out of a reader whose + callers only ever catch ``ContextError``, so the graph-context path + would propagate it instead of reporting the graph unreadable. + """ + for field_name, record in (("file_type", "nodes"), ("confidence", "links")): + for value in ([], {}, ["code"], {"value": "EXTRACTED"}, 3, None, True): + with self.subTest(field=field_name, value=value): + document = graph_document() + document[record][0][field_name] = value + with self.assertRaises(ContextError): + self.load(document) + def test_refuses_an_unreadable_source_location(self) -> None: for location in ("12", "line 12", "L", "L0", "L-4", "L99999999999"): with self.subTest(location=location): @@ -521,6 +549,24 @@ def test_an_out_of_scope_citation_is_refused(self) -> None: class PacketTests(GraphWorkspace): + def test_a_packet_binds_the_generation_its_evidence_actually_came_from(self) -> None: + """An envelope authorizing another generation is refused, not copied. + + The delivery contract's freshness rules all read the binding's + generation, so a packet that carried an authorized-but-unqueried + generation would keep passing them after a rebuild -- the one case the + rules exist to catch. Nothing here rewrites the envelope: it is an + authorization this module did not mint. + """ + republished = self.publish(graph_document()) + self.assertNotEqual(republished.generation, self.manifest.generation) + # An authorization minted before the rebuild: it names the generation + # that is gone, while the traversal reads the one published now. + outcome = self.context(envelope=self.authorized(generation=self.manifest.generation)) + self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(outcome.summary["reason"], "uncitable") + self.assertIsNone(outcome.packet) + def test_packet_carries_its_provenance_and_loads_through_the_contract(self) -> None: outcome = self.context() self.assertEqual(outcome.status, query.AVAILABLE) @@ -665,6 +711,27 @@ def test_a_generation_without_a_provider_export_is_unreadable(self) -> None: self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) self.assertEqual(outcome.summary["reason"], "unreadable") + def test_a_malformed_vocabulary_field_reports_unreadable_both_ways(self) -> None: + """Required blocks and optional degrades, which is what "unreadable" means. + + The reader refuses a ``file_type`` that is a JSON array rather than a + word, and this is the path that refusal has to arrive on: a graph the + caller is told it cannot use, not an exception out of an opt-in + feature. Both dispositions are covered because only one of them pauses + the dependent work. + """ + document = graph_document() + document["nodes"][0]["file_type"] = ["code"] + self.publish(document) + blocked = self.context() + self.assertEqual(blocked.status, query.REQUIRED_UNAVAILABLE) + self.assertEqual(blocked.summary["reason"], "unreadable") + self.assertEqual(blocked.dependent_work, "paused") + degraded = self.context(policy=policy(required=False)) + self.assertEqual(degraded.status, query.OPTIONAL_UNAVAILABLE) + self.assertEqual(degraded.summary["reason"], "unreadable") + self.assertEqual(degraded.dependent_work, "usable") + def test_an_unresolved_target_blocks_required_context(self) -> None: outcome = self.context(target="no_such_symbol") self.assertEqual(outcome.status, query.REQUIRED_UNAVAILABLE) @@ -684,7 +751,7 @@ def authorization(self, **overrides) -> Path: # rather than at the fixed ``NOW`` the library-level tests use. live = datetime.now(timezone.utc) + timedelta(minutes=30) payload = { - "connection": envelope(expires_at=live.isoformat()), + "connection": self.authorized(expires_at=live.isoformat()), "policy": policy(), "repository": "owner/repo", "work_item": "work-item-one", From db30846eed2144ef58f1a3cf5608666de9d48752 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 01:11:52 -0700 Subject: [PATCH 08/33] Graph context: require stated direction and the consuming revision Two merge blockers from the audit at 7ec69d5. Direction is no longer inferred. The pinned build stores an undirected NetworkX graph by default, where source/target are an endpoint pair the storage ordered rather than a caller and a callee. The exporter tries to repair that from its _src/_tgt stash, but the written link keeps no record that the repair happened, so a reader cannot tell a restored edge from one ordered by node iteration. Every answer here is an oriented claim, so a generation that does not declare itself directed is refused rather than answered from. Authorization and attachment now bind the consuming revision. The graph was authorized against the registered checkout's HEAD, so evidence for commit A passed for work at commit B. The consuming revision travels on ContextRequest.revision: prepare and guided delivery read it from the session's own checkout, attachment and replay use the trusted current PR head, and a repository-kind load that cannot name one is refused rather than falling back. Organization context is untouched -- its source revision is a document version with no reason to equal a code commit. Regressions cover an undirected export, retrieval and replay against a revision the graph was not built from, a load with no consuming revision, and attachment to a PR head other than the packet's. Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_command.py | 5 +- src/code_mower/context_delivery.py | 30 ++++++- src/code_mower/context_graph_connection.py | 32 +++++++- src/code_mower/context_graph_query.py | 32 ++++++++ src/code_mower/context_guided.py | 9 ++- src/code_mower/context_packets.py | 92 ++++++++++++++++++---- src/code_mower/context_prepare.py | 10 ++- src/code_mower/devin_work_orders.py | 7 +- tests/test_context_graph_connection.py | 74 ++++++++++++++--- tests/test_context_graph_query.py | 25 +++++- 10 files changed, 276 insertions(+), 40 deletions(-) diff --git a/src/code_mower/context_command.py b/src/code_mower/context_command.py index dcde73b3..8a220dfd 100644 --- a/src/code_mower/context_command.py +++ b/src/code_mower/context_command.py @@ -14,7 +14,7 @@ from .claude_audit_pr import _decision_authorities_for_repo from .context_contract import ContextError, ContextRequest, _object, normalize_policy from .context_delivery import SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, attach, deliver, read_binding, render_evidence -from .context_packets import load_authorized +from .context_packets import consuming_revision, load_authorized from .context_store import ContextStore, strict_json from .provider_runners import fetch_issue_comments, fetch_pull_request, post_pr_comment from .provider_runners.github_auth import resolve_github_token_from_env_or_gh @@ -63,7 +63,8 @@ def main(argv=None): if args.recipient not in SUPPORTED_RECIPIENTS or args.recipient.endswith(":reviewer"): raise ContextError("independent reviewers consume an attached review revision") packet = load_authorized(store, args.connection, args.packet, spec["policy"], - ContextRequest(spec["repository"], spec["work_item"], args.recipient)) + ContextRequest(spec["repository"], spec["work_item"], args.recipient, + consuming_revision(args.repo_path))) print(render_evidence(packet, args.packet), end="") return 0 if args.command in ('deliver', 'feedback') and (args.connection or args.request_stdin): diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index a85e9efa..543dfdb4 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -76,11 +76,17 @@ def read_binding(store, revision): return _binding(lookup.artifact("d-" + revision).read()) -def _packet_for_binding(store, binding, recipient, *, backend=None): +def _packet_for_binding(store, binding, recipient, *, backend=None, revision=None): if recipient not in SUPPORTED_RECIPIENTS: raise ContextError("this participant cannot consume private context in this release") + # The consuming revision of a delivery is the head the binding was published + # for, which the caller has already confirmed against the trusted current + # input. A repository-kind connection re-derives its authorization from that + # commit; an organization connection ignores it. + revision = revision or binding["metadata"]["head"] packet = load_authorized(store, binding["connection"], binding["handle"], binding["policy"], - ContextRequest(binding["repository"], binding["work_item"], recipient), backend=backend) + ContextRequest(binding["repository"], binding["work_item"], recipient), backend=backend, + revision=revision) if packet.sha256 != binding["packet_sha256"]: raise ContextError("context evidence changed; attach the new input and review again") return packet @@ -103,11 +109,21 @@ def reserve_attachment( A caller-supplied revision lets a guided session persist its intent before touching GitHub and resume that exact intent after a crash. Repeating the same reservation is idempotent; a conflicting reuse fails closed. + + ``revision`` here is that attachment handle, not a Git revision. The Git + revision an attachment binds is ``head``: the PR head the caller read from + the trusted remote. Repository-kind evidence is authorized and checked + against that commit, so a packet prepared while the checkout sat at commit A + cannot be attached to a PR whose head is commit B. """ if request.recipient not in SUPPORTED_RECIPIENTS or not request.recipient.endswith(":orchestrator"): raise ContextError("an approved orchestrator must attach context") policy = normalize_policy(policy) - packet = load_authorized(store, name, handle, policy, request, backend=backend) + packet = load_authorized( + store, name, handle, policy, + ContextRequest(request.repository, request.work_item, request.recipient, head), + backend=backend, revision=head, + ) payload = packet.private_payload() revision = revision or uuid.uuid4().hex _handle(revision) @@ -122,7 +138,13 @@ def reserve_attachment( # published generation is what a packet binds, so a graph rebuilt # between preparation and attachment fails the same check a revoked # organization authorization does. - generation = graph_connection.current_generation(state, root=store.root) + generation = graph_connection.current_generation(state, root=store.root, revision=head) + # Stated here as well as enforced on the load, because this is the + # line an attachment is read off: repository evidence describes one + # commit's code, and the commit this PR is at is the only one it may + # be attached to. + if payload["source_revision"] != head: + raise ContextError("context evidence is not bound to the current pull request head") else: state = _state(saved, name) generation = state["generation"] diff --git a/src/code_mower/context_graph_connection.py b/src/code_mower/context_graph_connection.py index 9fafa8bc..084b6c8e 100644 --- a/src/code_mower/context_graph_connection.py +++ b/src/code_mower/context_graph_connection.py @@ -172,6 +172,28 @@ def current_generation(state: Mapping[str, Any], *, root: Path | None = None, return manifest.generation +def authorized_revision(locked, name: str, *, root: Path | None = None, revision: str = "HEAD", + now: datetime | None = None) -> tuple[dict[str, Any], str]: + """One envelope, and the commit the graph is bound to for ``revision``. + + The commit is returned rather than re-derived by the caller because it is + the same ``graph_status`` read that authorized the load: asking twice would + rehash the artifact and, worse, could answer differently across a rebuild, + so the envelope and the revision a packet is checked against would then come + from two different observations of the graph. + + ``revision`` is the *consuming* revision -- the commit whose work the + evidence is for -- and not merely the registered checkout's ``HEAD``. A + graph that was not built from exactly that commit resolves ``stale`` in + ``_published`` and never reaches a recipient. + """ + current = now or datetime.now(timezone.utc) + state = saved_state(locked.read(), name) + manifest = _published(state, root=root, revision=revision).manifest + assert manifest is not None + return _envelope(state, name, manifest, current), manifest.commit + + def authorize_locked(locked, name: str, *, root: Path | None = None, revision: str = "HEAD", now: datetime | None = None) -> dict[str, Any]: """Mint one envelope from current local state, under the caller's lock. @@ -183,10 +205,11 @@ def authorize_locked(locked, name: str, *, root: Path | None = None, revision: s requested revision -- and what it publishes as ``generation`` is the graph's, so the shared packet contract does the rest. """ - current = now or datetime.now(timezone.utc) - state = saved_state(locked.read(), name) - manifest = _published(state, root=root, revision=revision).manifest - assert manifest is not None + return authorized_revision(locked, name, root=root, revision=revision, now=now)[0] + + +def _envelope(state: Mapping[str, Any], name: str, manifest: Any, + current: datetime) -> dict[str, Any]: envelope = { "schema": CONNECTION_SCHEMA, "capability_version": CAPABILITY_VERSION, "connection": name, "provider": PROVIDER, "kind": CONNECTION_KIND, @@ -261,6 +284,7 @@ def retrieve(state: Mapping[str, Any], spec: Mapping[str, Any], *, envelope: Map "GRAPH_SCHEMA", "PROVIDER", "authorize_locked", + "authorized_revision", "connect", "connection_spec", "current_generation", diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 82fa0513..4a29ffc8 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -76,6 +76,29 @@ #: acts on, and it is checked against the generation rather than trusted. GRAPH_EDGE_KEYS = ("links", "edges") +#: Whether the export preserves the direction of its relationships, which every +#: question here depends on and no question here can recover. +#: +#: The pinned build writes a NetworkX graph that is undirected by default +#: (``build.py::build_from_json(directed=False)``). Undirected storage +#: canonicalizes endpoint order, so ``source``/``target`` in an undirected +#: export are an endpoint *pair*, not a caller and a callee. The exporter does +#: try to repair that -- it stashes the true endpoints in ``_src``/``_tgt`` and +#: restores them before writing -- but the restored link carries no record that +#: the repair happened, so a reader cannot tell a restored edge from one whose +#: order came out of node iteration, and an undirected build additionally +#: collapses a pair related in both directions onto whichever it saw first. +#: +#: Every answer this module produces is an oriented claim: ``impact`` and +#: ``dependency`` are the same relationships walked in opposite directions, and +#: even a ``symbol`` neighbourhood states "A calls B" rather than "A and B are +#: adjacent". Reading orientation out of a document that does not establish it +#: is how a packet comes to assert the reverse of what the code does, so a +#: generation that does not declare itself directed is refused here rather than +#: answered from. ``directed: true`` is the provider's own statement that the +#: graph was stored as a ``DiGraph``, where source and target *are* the edge. +GRAPH_DIRECTED_KEY = "directed" + #: Required node and edge fields, taken from the pinned validator's #: ``REQUIRED_NODE_FIELDS`` and ``REQUIRED_EDGE_FIELDS``. A record missing one #: of these is a refusal: the provider's own validator would not have passed @@ -490,6 +513,15 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C raise ContextError("local graph node count exceeds its budget") if not isinstance(raw_edges, list) or len(raw_edges) > MAX_EDGES: raise ContextError("local graph edge count exceeds its budget") + # Before a single edge is read, because direction is not a property of any + # one link: an undirected export's endpoints are a pair the storage ordered, + # and no traversal, filter or sentence below can be honest about a + # relationship whose orientation the document never stated. + if payload.get(GRAPH_DIRECTED_KEY) is not True: + raise ContextError( + "local graph export does not preserve relationship direction; rebuild the " + "generation as a directed graph" + ) stamped = payload.get("built_at_commit") if stamped is not None and _text(stamped, maximum=64) != commit: raise ContextError("local graph was built from a different commit than its generation") diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index aee2504b..4c55407f 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -21,7 +21,7 @@ reserve_attachment, retire_attachment, ) -from .context_packets import load_authorized +from .context_packets import consuming_revision, load_authorized from .context_store import ContextStore from .participants import PARTICIPANTS, participant_id from .provider_runners import fetch_issue_comments, fetch_pull_request, post_pr_comment @@ -371,7 +371,12 @@ def deliver_session( record["connection"], record["packet"], record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest( + record["repo"], record["work_item"], recipient, + # Before attachment the consuming revision is this + # checkout's, which is what the builder is about to work on. + consuming_revision(repo_path), + ), backend=backend, ) text = render_evidence(packet, record["packet"]) diff --git a/src/code_mower/context_packets.py b/src/code_mower/context_packets.py index f4d3c6ee..662ee141 100644 --- a/src/code_mower/context_packets.py +++ b/src/code_mower/context_packets.py @@ -12,6 +12,7 @@ from pathlib import Path from . import context_graph_connection as graph_connection +from . import context_graph_lifecycle as lifecycle from .context_connections import _backend, _state, authorize_locked from .context_contract import ( CAPABILITY_VERSION, PACKET_SCHEMA, ContextError, ContextRequest, _object, @@ -42,6 +43,26 @@ def request_spec(value, name): "policy": policy} +def consuming_revision(repo_root) -> str | None: + """The commit the *consuming* checkout is at, or ``None`` if it has none. + + This is the revision prepared evidence is for, and it is read from the + checkout doing the work rather than from whichever checkout a connection was + registered against: the two are routinely different commits, and a local + repository graph describing the other one does not describe this work. + + ``None`` rather than a raise, so a connection that has no use for a code + revision -- an organization search, whose sources are documents with their + own versions -- still prepares from a directory that is not a Git checkout. + A repository-kind connection refuses instead of falling back. + """ + try: + commit, _tree = lifecycle.resolve_revision(Path(repo_root)) + except (ContextError, OSError, ValueError): + return None + return commit + + def _handle(value): try: if uuid.UUID(hex=value).hex != value: @@ -104,15 +125,33 @@ def _delete_entry(locked, entry): locked.artifact("p-" + entry["handle"]).delete() -def _request(spec, recipient=None): - return ContextRequest(spec["repository"], spec["work_item"], recipient or spec["recipient"]) +def _request(spec, recipient=None, revision=None): + return ContextRequest(spec["repository"], spec["work_item"], recipient or spec["recipient"], + revision) + + +def _load(store, entry, policy, request, envelope, *, bound_revision=None): + """Load one saved packet, and for a local graph require the consuming revision. + ``bound_revision`` is the commit the *consuming* work is at, resolved by the + same authorization that produced ``envelope``. The shared contract already + computes ``revision_state`` from it; what is decided here is what a + mismatch means. For a local repository graph it is a refusal: the evidence + describes one commit's code, so evidence for commit A handed to work on + commit B is wrong rather than merely old, and the caller's own required or + optional policy then decides whether that pauses or degrades the work. -def _load(store, entry, policy, request, envelope): + An organization connection passes ``None`` and is unaffected. Its source + revision is an external document version that has no reason to equal a code + commit, and requiring one would refuse every organization packet. + """ if entry["reference"] is None: raise ContextError("context retrieval did not complete; use an explicit refresh to try again") - return load_packet(private_root=store.root, reference=entry["reference"], policy=policy, - request=request, authorize=lambda: envelope) + packet = load_packet(private_root=store.root, reference=entry["reference"], policy=policy, + request=request, authorize=lambda: envelope) + if bound_revision is not None and packet.revision_state != "matching": + raise ContextError("local graph evidence is not bound to the consuming revision") + return packet def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revision="HEAD"): @@ -133,8 +172,14 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revis left = policy["timeout_seconds"] - (time.monotonic() - started) if left <= 0: raise ContextError("context retrieval deadline exceeded before authorization") + bound = None if local: - envelope = graph_connection.authorize_locked( + if revision is None: + # Default-deny rather than fall back to the registered + # checkout's ``HEAD``: that fallback is exactly how evidence for + # one commit reaches work on another. + raise ContextError("local graph context requires the consuming checkout revision") + envelope, bound = graph_connection.authorized_revision( locked, name, root=store.root, revision=revision, ) else: @@ -148,7 +193,8 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revis index_file, index = _index(locked) old = next((entry for entry in index["entries"] if entry["key"] == fingerprint), None) if old is not None and not refresh: - packet = _load(store, old, policy, _request(spec), envelope) + packet = _load(store, old, policy, _request(spec, revision=bound), envelope, + bound_revision=bound) return {**packet.shareable_summary(), "status": "available", "packet_handle": old["handle"], "reused": True, "usage": old["usage"]} # Reserve before any paid/read tool call. Restarting a failed attempt @@ -190,7 +236,8 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revis locked.artifact("p-" + entry["handle"]).write(packet_data) raw = json.dumps(packet_data, allow_nan=False, separators=(",", ":")).encode() entry["reference"] = {"path": ".p-" + entry["handle"] + ".json", "sha256": hashlib.sha256(raw).hexdigest()} - packet = _load(store, entry, policy, _request(spec), envelope) + packet = _load(store, entry, policy, _request(spec, revision=bound), envelope, + bound_revision=bound) entry["usage"] = usage index_file.write(index) _index(locked) @@ -209,28 +256,43 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revis def load_authorized(store, name, handle, policy, request: ContextRequest, *, backend=None, - revision="HEAD"): + revision=None): """Every participant replay obtains a new authorization under lock. For an organization connection that is a fresh online check. For a local repository graph it is a fresh read of current local state: the published - generation for ``revision``. Either way the envelope is minted here and - now, so a packet whose graph was rebuilt or whose revision has moved on is - refused by the shared contract rather than replayed. + generation for the *consuming* revision. Either way the envelope is minted + here and now, so a packet whose graph was rebuilt or whose revision has + moved on is refused by the shared contract rather than replayed. + + The consuming revision travels on the request the caller already builds -- + ``ContextRequest.revision`` -- and ``revision`` is the same value for the + callers that hold it without holding a request, such as an attachment that + knows only the trusted current PR head. Neither is defaulted to ``HEAD`` + for a graph: a replay that cannot name the revision it is for is refused, + because the checkout the graph was registered from moves independently of + the work consuming the evidence. """ _handle(handle) with store.locked(name) as locked: + bound = None if graph_connection.is_graph(locked.read()): - envelope = graph_connection.authorize_locked( - locked, name, root=store.root, revision=revision, + consuming = request.revision or revision + if consuming is None: + raise ContextError("local graph context requires the consuming checkout revision") + envelope, bound = graph_connection.authorized_revision( + locked, name, root=store.root, revision=consuming, ) + # Resolved, so a symbolic consuming revision is compared as the + # commit it names rather than as the word the caller typed. + request = ContextRequest(request.repository, request.work_item, request.recipient, bound) else: envelope = authorize_locked(locked, name, backend or _backend()) _file, index = _index(locked) entry = next((entry for entry in index["entries"] if entry["handle"] == handle), None) if entry is None: raise ContextError("context packet is missing or was invalidated") - return _load(store, entry, policy, request, envelope) + return _load(store, entry, policy, request, envelope, bound_revision=bound) def main(argv=None): diff --git a/src/code_mower/context_prepare.py b/src/code_mower/context_prepare.py index 328e19b8..ae163269 100644 --- a/src/code_mower/context_prepare.py +++ b/src/code_mower/context_prepare.py @@ -9,6 +9,7 @@ from . import context_packets, context_session, work_orders from .context_contract import ContextError, ContextRequest, _text +from .context_packets import consuming_revision from .context_delivery import SUPPORTED_HOSTS from .context_store import ContextStore from .participants import PARTICIPANTS, participant_id @@ -240,6 +241,10 @@ def prepare( reused=True, ), 0 + # Read once, before any retrieval or replay: every load below is for the + # work this checkout is at, and a checkout that moves mid-preparation must + # not have one packet authorized against two commits. + revision = consuming_revision(repo_root) explicit_query = query is not None effective_query = _text( query if explicit_query else DEFAULT_QUERY_PREFIX + record["work_item"], @@ -287,7 +292,7 @@ def prepare( record["connection"], record["packet"], record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest(record["repo"], record["work_item"], recipient, revision), backend=backend, ) except ContextError as exc: @@ -379,7 +384,7 @@ def prepare( record["connection"], packet_handle, record["policy"], - ContextRequest(record["repo"], record["work_item"], recipient), + ContextRequest(record["repo"], record["work_item"], recipient, revision), backend=backend, ) except ContextError as exc: @@ -407,6 +412,7 @@ def prepare( }, backend=backend, refresh=refresh, + revision=revision, ) except ContextError as exc: context_session.record_failure(association_store, record, exc) diff --git a/src/code_mower/devin_work_orders.py b/src/code_mower/devin_work_orders.py index 6b9266bc..1bd9ba07 100644 --- a/src/code_mower/devin_work_orders.py +++ b/src/code_mower/devin_work_orders.py @@ -251,6 +251,10 @@ class PacketContext: handle: str = field(repr=False) policy: dict = field(repr=False) backend: object = field(default=None, repr=False) + #: The commit this order's work consumes. Repository-kind evidence is + #: authorized against it and refuses when it is absent, because a graph + #: describing another commit does not describe this order. + revision: str | None = field(default=None, repr=False) def packet_context(store: ContextStore, name: str, handle: str, policy, *, order: WorkOrder, @@ -362,7 +366,8 @@ def _evidence(order, context): if (order.context_policy == "none" or type(context) is not PacketContext or type(context.store) is not ContextStore): raise RemoteError("context_binding_mismatch") - request = ContextRequest(order.repository, order.work_item, CONTEXT_RECIPIENT) + request = ContextRequest(order.repository, order.work_item, CONTEXT_RECIPIENT, + context.revision) try: packet = load_authorized(context.store, context.name, context.handle, context.policy, request, backend=context.backend) diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index 99ca236d..732c8e49 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -96,13 +96,15 @@ def spec(self, *, question: str = "impact", target: str = "parse_config", "policy": {**POLICY, "required": required}, } - def fetch(self, **overrides): - return context_packets.fetch(self.store, "local-graph", self.spec(**overrides)) + def fetch(self, *, revision: str = "HEAD", **overrides): + return context_packets.fetch( + self.store, "local-graph", self.spec(**overrides), revision=revision, + ) - def load(self, handle: str, recipient: str): + def load(self, handle: str, recipient: str, revision: str = "HEAD"): return context_packets.load_authorized( self.store, "local-graph", handle, POLICY, - ContextRequest("owner/repo", "WORK-1", recipient), + ContextRequest("owner/repo", "WORK-1", recipient, revision), ) # -- retrieval through the shared store ------------------------------ @@ -192,6 +194,48 @@ def test_an_explicitly_named_prior_revision_still_loads_its_own_packet(self) -> ) self.assertEqual(packet.private_payload()["source_revision"], self.manifest.commit) + def _second_commit_the_checkout_is_not_on(self) -> str: + """Make a commit, then leave the registered checkout back on the first. + + This is the arrangement the consuming-revision rule exists for: the + connected checkout still sits at the commit its graph was built from, + while the work consuming the evidence is at another commit entirely -- + a builder's branch, a PR head, a worktree. Both commits are real and + resolvable, so nothing here fails for want of an object. + """ + first = lifecycle.resolve_revision(self.repository)[0] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "consuming work") + second = lifecycle.resolve_revision(self.repository)[0] + git(self.repository, "reset", "-q", "--hard", first) + self.assertEqual(lifecycle.resolve_revision(self.repository)[0], self.manifest.commit) + self.assertNotEqual(second, self.manifest.commit) + return second + + def test_retrieval_refuses_a_graph_that_is_not_the_consuming_revisions(self) -> None: + """The registered checkout's ``HEAD`` is not the revision being worked on.""" + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self.fetch(revision=consuming) + + def test_replay_refuses_a_packet_for_another_revisions_code(self) -> None: + handle = self.fetch()["packet_handle"] + consuming = self._second_commit_the_checkout_is_not_on() + # The checkout's own HEAD still authorizes, which is exactly why the + # consuming revision has to be the one asked about. + self.assertEqual(self.load(handle, "claude:builder").revision_state, "matching") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder", revision=consuming) + + def test_a_load_that_cannot_name_its_consuming_revision_is_refused(self) -> None: + handle = self.fetch()["packet_handle"] + with self.assertRaises(ContextError): + context_packets.load_authorized( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder"), + ) + def test_removing_the_graph_makes_the_connection_unavailable(self) -> None: handle = self.fetch()["packet_handle"] lifecycle.remove_graph(self.repository, root=self.private) @@ -287,15 +331,27 @@ def test_reuse_after_a_rebuild_pauses_required_work(self) -> None: report, code = self.prepare(saved) self.assertEqual((code, report["status"]), (1, "required_unavailable")) + def _attach(self, handle: str, head: str): + return context_delivery.reserve_attachment( + self.store, "local-graph", handle, POLICY, + ContextRequest("owner/repo", "WORK-1", "codex:orchestrator"), + pr=1, head=head, + ) + def test_attachment_refuses_a_packet_whose_graph_was_rebuilt(self) -> None: handle = self.fetch()["packet_handle"] self.publish() with self.assertRaises(ContextError): - context_delivery.reserve_attachment( - self.store, "local-graph", handle, POLICY, - ContextRequest("owner/repo", "WORK-1", "codex:orchestrator"), - pr=1, head="c" * 40, - ) + self._attach(handle, self.manifest.commit) + + def test_attachment_binds_the_pull_requests_head_not_the_checkouts(self) -> None: + """A PR at another commit cannot carry this commit's graph evidence.""" + handle = self.fetch()["packet_handle"] + # The head the graph *is* for attaches. + self.assertEqual(self._attach(handle, self.manifest.commit)["head"], self.manifest.commit) + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self._attach(handle, consuming) @unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index e96dec1f..11f9726f 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -131,7 +131,9 @@ def graph_document(**extra) -> dict: for each indexed file. """ return { - "directed": False, + # Directed, because every question this module answers is an oriented + # claim and an undirected export's endpoint order is storage order. + "directed": True, "multigraph": False, "graph": {}, "nodes": [ @@ -371,6 +373,27 @@ def test_refuses_a_graph_built_from_another_commit(self) -> None: # Agreeing is fine, and is the ordinary case. self.assertEqual(len(self.load(graph_document(built_at_commit="b" * 40)).nodes), 5) + def test_refuses_an_export_that_does_not_preserve_direction(self) -> None: + """An undirected export states an endpoint pair, not a caller and callee. + + The provider's undirected storage canonicalizes endpoint order and its + export's repair leaves no mark a reader can check, so every oriented + answer here -- ``impact``, ``dependency``, and the ``calls`` sentence a + ``symbol`` neighbourhood states -- would be asserting an orientation the + document never established. The refusal names direction, so an operator + reads it as "rebuild directed" rather than as a corrupt graph. + """ + for flag in (False, None, "true", 1): + with self.subTest(directed=flag): + document = graph_document() + if flag is None: + document.pop("directed") + else: + document["directed"] = flag + with self.assertRaises(ContextError) as caught: + self.load(document) + self.assertIn("direction", str(caught.exception)) + def test_refuses_records_missing_the_providers_required_fields(self) -> None: for mutate in ( lambda doc: doc["nodes"][0].pop("label"), From 94cf43f661d1127753247e6f1d82924da3bcb292 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 01:26:49 -0700 Subject: [PATCH 09/33] Graph context: consume the actual raw no-cluster extraction The blanket directed:true requirement made the previous head incompatible with every generation the lifecycle can actually produce. The lifecycle requires extract --code-only --no-cluster, and at the pinned commit that branch of the CLI dumps the merged extractor result straight to graph.json through write_json_atomic: it never builds a NetworkX graph, never calls export.py::to_json, and so writes no directed, multigraph, graph, links or built_at_commit key. Its edge source/target are the endpoints the extractor's own add_edge recorded at the call site, not an undirected container's iteration order. The reader now distinguishes the two documents structurally, by the presence of a node-link marker key rather than by the name of the edge list -- keying off "edges" would misread a newer NetworkX export, which renamed links to edges, as a raw extraction and skip the direction requirement it needs. A raw extraction loads with its endpoint direction preserved and no marker demanded. A node-link export is still read, and is still refused unless it declares directed:true, because that path's endpoints did pass through storage that may have canonicalized them. The misleading compatibility fixture is replaced with a representative raw extraction shape, including the absence of a directed marker, and the node-link shape moves to its own fixture and test class so no acceptance claim rests on editing one into the other. New evidence: a raw document loads with no marker, reversing the node list does not reverse a raw extractor edge, each marker alone identifies the node-link format, and the renamed edges key is still held to the direction requirement. Docs corrected to the actual supported path, including the false claim that pruned records are counted -- they are pruned and not counted, and what a packet says about its own incompleteness is the traversal's truncation and omission fields. The consuming-revision work from the previous head is preserved, and its one untested consumer is repaired: PacketContext grew a revision field there without updating the Devin field-shape assertion, and packet_context() had no parameter for it, so the hosted builder could never supply the commit its repository-kind evidence is authorized against. The factory now takes and validates one, with regressions for the field shape, the validation, and the revision reaching ContextRequest. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 100 +++++++--- src/code_mower/context_graph_query.py | 153 ++++++++++----- src/code_mower/devin_work_orders.py | 14 +- tests/test_context_graph_query.py | 267 +++++++++++++++++++------- tests/test_devin_work_orders.py | 42 +++- 5 files changed, 433 insertions(+), 143 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 705cfdfb..da9fce13 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -18,58 +18,101 @@ Nothing here installs, downloads, or runs a provider, and nothing here is on a default path. The only subprocess is Git, reading blobs of the commit the generation is already bound to. -## Reading the pinned provider's own export +## Reading the pinned provider's own output A generation's artifact is the provider's own state, packed reproducibly. This -adapter reads one member of it, `graph.json` — the file the pinned Graphify -release writes from `graphify/export.py::to_json`. There is no Code Mower graph +adapter reads one member of it, `graph.json`. There is no Code Mower graph schema and no normalization pass between the build and the query: the lifecycle archives what the provider wrote, so this is what gets read. +### The supported document: the raw `--no-cluster` extraction + +`context_graph_lifecycle` runs the pinned release as +`extract --code-only --no-cluster` and accepts no other options +(`_REQUIRED_EXTRACT_OPTIONS`). At the pinned commit that branch of +`graphify/cli.py` dumps the merged extractor result straight to `graph.json` +through `write_json_atomic`. It never builds a NetworkX graph and never calls +`export.py::to_json`, so the document carries **no** `directed`, `multigraph`, +`graph`, `links` or `built_at_commit` key: + ```json { - "directed": false, "multigraph": false, "graph": {}, "nodes": [ {"id": "n-config", "label": "parse_config", "file_type": "code", "source_file": "example_pkg/config.py", "source_location": "L12", - "community": 0, "norm_label": "parse_config"} + "type": "function", "metadata": {"namespace": "example_pkg"}} ], - "links": [ + "edges": [ {"source": "n-load", "target": "n-config", "relation": "calls", "confidence": "EXTRACTED", "source_file": "example_pkg/loader.py", - "source_location": "L41", "weight": 1.0, "confidence_score": 1.0} + "source_location": "L41", "weight": 1.0} ], "hyperedges": [], - "built_at_commit": "…" + "input_tokens": 0, "output_tokens": 0, + "extracted_sources": ["example_pkg/config.py", "example_pkg/loader.py"] } ``` -What is validated is the *provider's* contract, not one of ours: +Direction here is the extractor's own claim. `engine.py::add_edge` writes the +call site's `source` and `target`, and the raw dump carries that record through +unchanged, so reading the edge record reads the provider's semantics. Permuting +the node list cannot reverse a relationship, because nothing about the endpoint +order is derived from node order. + +### The other document: a NetworkX node-link export + +A generation built through the clustered path instead carries +`export.py::to_json`'s output — `networkx.json_graph.node_link_data`, which +always writes `directed`, `multigraph`, `graph` and a `links` list (NetworkX +later renamed that key to `edges`; the pinned validator accepts either). That +document is read too, but it is held to one extra requirement the raw format +does not need and does not get: + +> **`directed` must be `true`.** The clustered build stores an `nx.Graph` by +> default, and undirected storage canonicalizes endpoint order. `to_json` tries +> to repair this — the build stashes the true endpoints in `_src`/`_tgt` and the +> exporter restores them — but the written link carries no record that the +> repair happened, so a reader cannot distinguish a restored edge from one +> ordered by node iteration, and an undirected build additionally collapses a +> pair related in both directions onto whichever it saw first. Every answer this +> module produces is an oriented claim, so an undirected node-link export is +> refused rather than answered from. + +The two are told apart structurally, by the presence of a node-link marker key, +never by guessing from the name of the edge list — otherwise a newer NetworkX +export that names its links `edges` would be misread as a raw extraction and +skip the direction requirement it needs. + +`built_at_commit` is likewise a node-link-only stamp. Where it is present it +must equal the commit the generation is bound to; the raw path writes none, and +there the binding rests on the lifecycle's own commit binding and on the census +check every citation goes through. + +What is validated is the *provider's* contract, not one of ours, and it is the +same for both documents: - The required node and edge fields of `graphify/validate.py` — `id`, `label`, `file_type`, `source_file` on a node; `source`, `target`, `relation`, - `confidence`, `source_file` on a link. A record missing one would not have + `confidence`, `source_file` on an edge. A record missing one would not have passed the provider's own validator, so it is a refusal here. - Its vocabularies. `file_type` must be one of the six it defines, and `confidence` must be uppercase `EXTRACTED`/`INFERRED`/`AMBIGUOUS`. Lowercase is the *packet* vocabulary, and a graph using it was not written by the - pinned exporter. + pinned provider. - Its locations. `source_location` is `L` or empty; anything else is a location this module could not check against the bound commit, so it refuses - rather than traversing past it. One line per node, never a span: the export - records no extent, and claiming one would be this adapter inventing it. -- `built_at_commit`, when the exporter stamped it, must equal the commit the - generation is bound to. Otherwise the artifact and the manifest describe - different revisions. - -Three things are deliberately *not* refusals, because the real export carries -them and rejecting them would reject every ordinary generation: - -- **Extra annotations.** The exporter adds `community`, `community_name` and - `norm_label` to nodes and `confidence_score` to links; the extractor adds - `weight`, `context`, `type` and a free-form `metadata` dict from an LLM - extraction. None of them changes a traversal, so none is read. Everything - this module *does* read is read by name and bounded. + rather than traversing past it. One line per node, never a span: neither + document records an extent, and claiming one would be this adapter inventing + it. + +Three things are deliberately *not* refusals, because a real generation carries +them and rejecting them would reject every ordinary one: + +- **Extra annotations.** The extractor adds `weight`, `context`, `type` and a + free-form `metadata` dict from an LLM extraction; a node-link export adds + `community`, `community_name` and `norm_label` to nodes and + `confidence_score` to links. None of them changes a traversal, so none is + read. Everything this module *does* read is read by name and bounded. - **Relations outside the mapped set.** The provider's validator does not constrain `relation` at all. Mapped relations (`calls`, `imports`, `defines`, `contains`, `references`, `inherits`, `implements`, `tests`) decide which @@ -80,8 +123,11 @@ them and rejecting them would reject every ordinary generation: - **Sourceless stubs and non-code corpora.** The extractor emits nodes with an empty `source_file` for cross-file references it could not resolve; those stay traversable and are never cited. Nodes whose `file_type` is not `code` - are dropped, and links onto a dropped node are pruned — which is the pinned - exporter's own treatment in `prune_dangling_edges`. + are dropped, and edges onto a dropped node are pruned — which is the pinned + exporter's own treatment in `prune_dangling_edges`. No count of the dropped + records is kept or reported: what a packet says about its own incompleteness + is the traversal's truncation and omission fields, not a tally of corpora + this module never queries. Node kinds — `file`, `symbol`, `test` — are **derived**, not read: a Graphify node declares its corpus and, rarely, a `type`, but never whether it is a file, diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 4a29ffc8..d3fbd94c 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -57,29 +57,53 @@ from .context_graph import MAX_GRAPH_CITATIONS, parse_graph_citation #: The graph document Code Mower reads. This is the pinned provider's own -#: ``graph.json`` -- the file ``graphify/export.py::to_json`` writes at commit -#: ``23f2ffa`` (release 0.9.58), which is the release the lifecycle (#913) pins -#: and archives. There is no Code Mower graph schema and no normalization step -#: between the two: an adapter that required a shape the build never produces -#: would reject every real generation, so this module reads the provider's -#: actual export and does the narrowing itself. +#: ``graph.json`` at commit ``23f2ffa`` (release 0.9.58), which is the release +#: the lifecycle (#913) pins and archives. There is no Code Mower graph schema +#: and no normalization step between the two: an adapter that required a shape +#: the build never produces would reject every real generation, so this module +#: reads the provider's actual output and does the narrowing itself. GRAPH_MEMBER = "graph.json" QUERY_SCHEMA = "code_mower.contextGraphQuery.v1" -#: The provider's export is a NetworkX ``node_link_data`` document: ``nodes`` -#: plus ``links``. ``edges`` is the same list under the name NetworkX used -#: before 3.2, and the pinned validator accepts either, so this reader does -#: too. Other top-level keys the pinned exporter writes -- ``directed``, -#: ``multigraph``, ``graph``, ``hyperedges``, ``built_at_commit`` -- are -#: provider bookkeeping; only ``built_at_commit`` carries a claim this module -#: acts on, and it is checked against the generation rather than trusted. -GRAPH_EDGE_KEYS = ("links", "edges") - -#: Whether the export preserves the direction of its relationships, which every -#: question here depends on and no question here can recover. +#: The two documents that can appear under ``graph.json``, which are *not* the +#: same file in two dialects and are not read as though they were. #: -#: The pinned build writes a NetworkX graph that is undirected by default +#: ``raw_extraction`` is what the lifecycle's own pinned invocation writes. +#: ``context_graph_lifecycle`` requires ``extract --code-only --no-cluster`` +#: (``_REQUIRED_EXTRACT_OPTIONS``), and the pinned CLI's ``--no-cluster`` branch +#: dumps the merged extractor result directly -- ``nodes``, ``edges``, +#: ``hyperedges``, token counts, ``extracted_sources`` -- through +#: ``write_json_atomic``. It never builds a NetworkX graph, never calls +#: ``to_json``, and therefore writes no ``directed``, ``multigraph``, ``graph`` +#: or ``built_at_commit`` key. Its ``source``/``target`` are the endpoints the +#: extractor's own ``add_edge`` recorded at the call site, so the orientation is +#: the provider's semantic claim and is preserved by reading the edge record. +#: +#: ``node_link`` is ``graphify/export.py::to_json``: a NetworkX +#: ``node_link_data`` document, which always carries ``directed``, +#: ``multigraph``, ``graph`` and ``links``. Its direction is *not* self-evident +#: and is handled separately, below. +GRAPH_FORMAT_RAW = "raw_extraction" +GRAPH_FORMAT_NODE_LINK = "node_link" + +#: Top-level keys only a NetworkX ``node_link_data`` document carries. Any one +#: of them means the file came through ``to_json`` rather than the raw +#: ``--no-cluster`` dump, and it is then read under the node-link rules -- +#: including the direction requirement -- whichever key names its edge list. +NODE_LINK_MARKERS = ("links", "directed", "multigraph", "graph") + +#: The edge list, per format. The raw dump writes ``edges``. ``node_link_data`` +#: writes ``links`` at the pinned commit (``to_json`` passes ``edges="links"``) +#: but NetworkX renamed the key, and the pinned validator accepts either, so a +#: node-link document is read under both names. +RAW_EDGE_KEY = "edges" +NODE_LINK_EDGE_KEYS = ("links", "edges") + +#: Whether a **node-link** export preserves the direction of its relationships, +#: which every question here depends on and no question here can recover. +#: +#: The clustered build writes a NetworkX graph that is undirected by default #: (``build.py::build_from_json(directed=False)``). Undirected storage #: canonicalizes endpoint order, so ``source``/``target`` in an undirected #: export are an endpoint *pair*, not a caller and a callee. The exporter does @@ -92,11 +116,16 @@ #: Every answer this module produces is an oriented claim: ``impact`` and #: ``dependency`` are the same relationships walked in opposite directions, and #: even a ``symbol`` neighbourhood states "A calls B" rather than "A and B are -#: adjacent". Reading orientation out of a document that does not establish it -#: is how a packet comes to assert the reverse of what the code does, so a -#: generation that does not declare itself directed is refused here rather than -#: answered from. ``directed: true`` is the provider's own statement that the -#: graph was stored as a ``DiGraph``, where source and target *are* the edge. +#: adjacent". Reading orientation out of a *node-link* document that does not +#: establish it is how a packet comes to assert the reverse of what the code +#: does, so such a document is refused rather than answered from. +#: ``directed: true`` is the provider's own statement that the graph was stored +#: as a ``DiGraph``, where source and target *are* the edge. +#: +#: This check belongs to the node-link format alone. Demanding the marker of a +#: raw extraction would refuse every generation the lifecycle's own pinned +#: options actually produce, since that path emits no marker and has no +#: undirected container to have lost direction in. GRAPH_DIRECTED_KEY = "directed" #: Required node and edge fields, taken from the pinned validator's @@ -408,9 +437,11 @@ def _node(value: Any) -> GraphNode | None: A non-``code`` node is dropped rather than refused. The provider indexes documents, papers, images, rationales and concepts into the same graph, and those are not repository relationships: they carry no location in the bound - commit, so no traversal here could cite one. Dropping them is bounded and - visible -- every edge that named one becomes a dangling edge, which - ``load_graph`` prunes and counts. + commit, so no traversal here could cite one. Dropping them is bounded -- + every edge that named one becomes a dangling edge, which ``load_graph`` + prunes. No count of the dropped records is kept or reported: what a packet + states about its own incompleteness is the traversal's truncation and + omission fields, not a tally of corpora this module never queries. """ record = _required(value, GRAPH_NODE_FIELDS, what="node") # Bounded text before membership: a vocabulary field is looked up in a set, @@ -473,6 +504,23 @@ def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge | None: ) +def _graph_format(payload: Mapping[str, Any]) -> str: + """Which of the provider's two ``graph.json`` documents this is. + + The discriminator is the presence of a NetworkX node-link marker, not the + name of the edge list. ``node_link_data`` always writes ``directed``, + ``multigraph`` and ``graph`` alongside its links, and the raw + ``--no-cluster`` dump writes none of them -- it writes the extractor's own + merged result, whose only structural keys are ``nodes`` and ``edges``. + Keying off ``edges`` instead would misread a newer NetworkX node-link + document, which names its links ``edges``, as a raw extraction and so skip + the direction requirement the node-link format needs. + """ + if any(key in payload for key in NODE_LINK_MARKERS): + return GRAPH_FORMAT_NODE_LINK + return GRAPH_FORMAT_RAW + + def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdge, ...]]: """Adjacency in one fixed order, so a traversal cannot depend on input order.""" buckets: dict[str, list[GraphEdge]] = {} @@ -487,40 +535,53 @@ def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdg def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> CodeGraph: """Read the pinned provider's ``graph.json`` into a bounded queryable graph. - The document is the pinned exporter's output, so what is validated here is - the provider's own contract -- the required fields of its validator, its + The document is the pinned provider's own output, so what is validated here + is the provider's own contract -- the required fields of its validator, its ``file_type`` and ``confidence`` vocabularies, its ``L`` locations -- and not a shape Code Mower invented. Every value this module reads is read by name and bounded; every value it does not read is left alone. + The *supported* document is the raw extraction the lifecycle's pinned + ``extract --code-only --no-cluster`` writes. A NetworkX node-link export is + also read, because a generation may have been produced by the clustered + path, but its provenance differs and it is held to the extra direction + requirement that path needs. The two are told apart structurally in + ``_graph_format``, never by guessing from an edge key. + Two provenance checks are worth more than any field check. The first is - ``built_at_commit``: the exporter stamps the commit the graph was built + ``built_at_commit``: the *exporter* stamps the commit the graph was built from, and if that disagrees with the commit the generation is bound to then the artifact and the manifest describe different revisions, which is a - refusal no traversal should be run past. The second is the census check - every citation goes through later. + refusal no traversal should be run past. The raw path writes no such stamp + -- it bypasses ``to_json`` entirely -- so for it this check is vacuous and + the binding rests on the lifecycle's own commit binding and on the second + check: the census every citation goes through later. """ if not isinstance(payload, Mapping): raise ContextError("local graph document must be an object") + graph_format = _graph_format(payload) raw_nodes = payload.get("nodes") - raw_edges = next( - (payload[key] for key in GRAPH_EDGE_KEYS if key in payload), - None, - ) + if graph_format == GRAPH_FORMAT_RAW: + raw_edges = payload.get(RAW_EDGE_KEY) + else: + raw_edges = next((payload[key] for key in NODE_LINK_EDGE_KEYS if key in payload), None) if raw_edges is None or raw_nodes is None: - raise ContextError("local graph document carries no provider nodes and links") + raise ContextError("local graph document carries no provider nodes and edges") if not isinstance(raw_nodes, list) or len(raw_nodes) > MAX_NODES: raise ContextError("local graph node count exceeds its budget") if not isinstance(raw_edges, list) or len(raw_edges) > MAX_EDGES: raise ContextError("local graph edge count exceeds its budget") # Before a single edge is read, because direction is not a property of any - # one link: an undirected export's endpoints are a pair the storage ordered, - # and no traversal, filter or sentence below can be honest about a - # relationship whose orientation the document never stated. - if payload.get(GRAPH_DIRECTED_KEY) is not True: + # one link: an undirected node-link export's endpoints are a pair the + # storage ordered, and no traversal, filter or sentence below can be honest + # about a relationship whose orientation the document never stated. A raw + # extraction is exempt because its endpoints never passed through a NetworkX + # container at all -- see ``GRAPH_DIRECTED_KEY``. + if graph_format == GRAPH_FORMAT_NODE_LINK and payload.get(GRAPH_DIRECTED_KEY) is not True: raise ContextError( - "local graph export does not preserve relationship direction; rebuild the " - "generation as a directed graph" + "local graph node-link export does not preserve relationship direction; " + "rebuild the generation with the pinned --no-cluster extraction or as a " + "directed graph" ) stamped = payload.get("built_at_commit") if stamped is not None and _text(stamped, maximum=64) != commit: @@ -531,9 +592,11 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C if node is None: continue if node.id in nodes: - # The provider's own validator does not check this, but its graph - # is a NetworkX node set and cannot hold two nodes under one id. A - # document that does was not written by the pinned exporter. + # The provider's own validator does not check this, but neither of + # its write paths can produce it: the raw dump runs its node list + # through ``build.dedupe_nodes`` and the clustered one through a + # NetworkX node set, and both collapse same-id nodes. A document + # that carries two was not written by the pinned provider. raise ContextError("local graph node identifiers must be unique") nodes[node.id] = node edges = tuple(sorted( diff --git a/src/code_mower/devin_work_orders.py b/src/code_mower/devin_work_orders.py index 1bd9ba07..c7c12b10 100644 --- a/src/code_mower/devin_work_orders.py +++ b/src/code_mower/devin_work_orders.py @@ -258,7 +258,7 @@ class PacketContext: def packet_context(store: ContextStore, name: str, handle: str, policy, *, order: WorkOrder, - backend=None) -> PacketContext: + backend=None, revision: str | None = None) -> PacketContext: """Bind one authorized packet to the hosted builder before its PR exists. The packet request is derived from the work order (repository and its @@ -266,17 +266,27 @@ def packet_context(store: ContextStore, name: str, handle: str, policy, *, order trusted policy's ``required`` flag must agree with the order's declared context policy. Each render performs a new online authorization for ``devin:builder``; nothing is cached or written. + + ``revision`` is the commit this order's work consumes. It is the one input + here a caller does supply, because there is nothing else to derive it from: + a hosted order has no PR head yet and this process is not the checkout doing + the work. Repository-kind evidence -- the local graph route among it -- is + authorized against that commit on every render, so without it such evidence + refuses rather than answering for whichever commit the connection happens to + be registered at. Organization-kind evidence, whose sources version + themselves, is unaffected and still prepares with ``None``. """ try: normalized = normalize_policy(policy) _handle(handle) + bound = _text(revision, maximum=200) if revision is not None else None except ContextError: raise RemoteError("invalid_request") from None if (normalized is None or order.context_policy == "none" or normalized["required"] != (order.context_policy == "required") or type(store) is not ContextStore or not isinstance(name, str)): raise RemoteError("invalid_request") - return PacketContext(store, name, handle, normalized, backend) + return PacketContext(store, name, handle, normalized, backend, bound) def _github_call(method, *args, **kwargs): diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 11f9726f..991bc6a4 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -120,36 +120,67 @@ def edge(source: str, target: str, relation: str, confidence: str = "EXTRACTED", } +def graph_nodes() -> list: + """A symbol, its caller, its caller's caller, a test, and a file node.""" + return [ + node("n-config", "parse_config", "example_pkg/config.py", 12), + node("n-load", "load", "example_pkg/loader.py", 40), + node("n-report", "render", "example_pkg/report.py", 5), + node("n-test", "test_parse_config", "tests/test_config.py", 8), + # The extractor's per-file node: label is the file's base name at L1. + node("n-config-file", "config.py", "example_pkg/config.py", 1), + ] + + +def graph_edges() -> list: + return [ + edge("n-load", "n-config", "calls"), + edge("n-report", "n-load", "calls", "INFERRED"), + edge("n-test", "n-config", "tests"), + edge("n-config-file", "n-config", "contains"), + ] + + def graph_document(**extra) -> dict: - """A small graph in the pinned provider's export format. - - Top-level shape is ``networkx.json_graph.node_link_data(G, edges="links")`` - as ``export.py::to_json`` writes it: ``directed``, ``multigraph``, - ``graph``, ``nodes``, ``links``, plus the ``hyperedges`` list and the - ``built_at_commit`` stamp the exporter appends. Contents are a symbol, its - caller, its caller's caller, a test, and the file node the extractor emits - for each indexed file. + """A small graph in the format the lifecycle's pinned invocation writes. + + ``context_graph_lifecycle`` requires ``extract --code-only --no-cluster``, + and the pinned CLI's ``--no-cluster`` branch dumps the merged extractor + result straight to ``graph.json`` through ``write_json_atomic``. So the + top-level shape is the raw extraction: ``nodes``, ``edges``, ``hyperedges``, + the token counters and ``extracted_sources``. + + What it deliberately does **not** carry is a ``directed`` marker, or + ``multigraph``, ``graph``, ``links`` or ``built_at_commit``. That path never + constructs a NetworkX graph and never calls ``export.py::to_json``, so none + of those keys exist in a real generation, and a fixture that added one to + satisfy the reader would be testing a file the provider never writes. + """ + return { + "nodes": graph_nodes(), + "edges": graph_edges(), + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + "extracted_sources": sorted(SOURCES), + **extra, + } + + +def node_link_document(**extra) -> dict: + """The same graph as ``export.py::to_json`` writes it, for the clustered path. + + ``networkx.json_graph.node_link_data(G, edges="links")`` plus the + ``hyperedges`` list and the ``built_at_commit`` stamp the exporter appends. + Directed, because this format's endpoint order is only a caller/callee claim + when the graph was stored as a ``DiGraph``. """ return { - # Directed, because every question this module answers is an oriented - # claim and an undirected export's endpoint order is storage order. "directed": True, "multigraph": False, "graph": {}, - "nodes": [ - node("n-config", "parse_config", "example_pkg/config.py", 12), - node("n-load", "load", "example_pkg/loader.py", 40), - node("n-report", "render", "example_pkg/report.py", 5), - node("n-test", "test_parse_config", "tests/test_config.py", 8), - # The extractor's per-file node: label is the file's base name at L1. - node("n-config-file", "config.py", "example_pkg/config.py", 1), - ], - "links": [ - edge("n-load", "n-config", "calls"), - edge("n-report", "n-load", "calls", "INFERRED"), - edge("n-test", "n-config", "tests"), - edge("n-config-file", "n-config", "contains"), - ], + "nodes": graph_nodes(), + "links": graph_edges(), "hyperedges": [], **extra, } @@ -278,12 +309,15 @@ def load(self, packet: dict, *, recipient: str, revision: str | None): class GraphSchemaTests(unittest.TestCase): - """The pinned provider's own export is what gets read, and read bounded. - - Every fixture in here is the shape ``graphify/export.py::to_json`` writes - at the pinned commit. The tests split into two halves on purpose: what the - real export carries must load, and what the provider's own validator would - reject must refuse. + """The pinned provider's own output is what gets read, and read bounded. + + The default fixture is the raw extraction the lifecycle's own pinned + ``extract --code-only --no-cluster`` writes. The node-link export the + clustered path writes is covered separately in ``NodeLinkFormatTests``, + because its provenance and its direction guarantee are different and must + not be tested by editing this fixture into that shape. The tests split into + two halves on purpose: what a real generation carries must load, and what + the provider's own validator would reject must refuse. """ def load(self, document: dict) -> query.CodeGraph: @@ -313,16 +347,52 @@ def test_keeps_the_providers_own_relation_and_lowercases_confidence(self) -> Non self.assertEqual((contains.relation, contains.kind), ("contains", "defines")) self.assertEqual(by_pair[("n-report", "n-load")].evidence, "inferred") - def test_reads_the_pre_3_2_edges_key(self) -> None: - """The pinned validator accepts ``edges`` for ``links``; so does this.""" + def test_reads_the_raw_extraction_without_a_directed_marker(self) -> None: + """The supported document declares no direction, and must not be asked to. + + The pinned ``--no-cluster`` branch writes the merged extractor result + directly: no NetworkX graph is built, ``to_json`` is never called, and + so ``directed``, ``multigraph``, ``graph`` and ``built_at_commit`` are + absent from every real generation. A reader that demanded the marker + would refuse the only path the lifecycle actually runs. + """ document = graph_document() - document["edges"] = document.pop("links") - self.assertEqual(len(self.load(document).edges), 4) + for absent in ("directed", "multigraph", "graph", "links", "built_at_commit"): + self.assertNotIn(absent, document) + graph = self.load(document) + self.assertEqual(len(graph.edges), 4) + # The orientation is the extractor's, and it is the one queried. + self.assertEqual( + {(item.source, item.target) for item in graph.edges if item.relation == "calls"}, + {("n-load", "n-config"), ("n-report", "n-load")}, + ) + + def test_reversed_node_iteration_does_not_reverse_a_raw_edge(self) -> None: + """A raw edge's endpoints come off the edge record, not from node order. + + This is the property the ``--no-cluster`` path has and an undirected + NetworkX container does not. ``add_edge`` writes the call site's own + source and target, so permuting the node list -- the only thing an + undirected container's endpoint order would follow -- cannot change + which way a relationship points. + """ + forward = self.load(graph_document()) + reversed_nodes = graph_document() + reversed_nodes["nodes"] = list(reversed(reversed_nodes["nodes"])) + permuted = self.load(reversed_nodes) + self.assertEqual( + [(item.source, item.target, item.relation) for item in forward.edges], + [(item.source, item.target, item.relation) for item in permuted.edges], + ) + # And the claim itself, stated the way a packet states it. + calls = next(item for item in permuted.edges if item.target == "n-config" + and item.relation == "calls") + self.assertEqual((calls.source, calls.target), ("n-load", "n-config")) def test_maps_an_unlisted_relation_without_asserting_a_listed_one(self) -> None: """An LLM-extracted relation is carried, grouped as ``related``, never renamed.""" document = graph_document() - document["links"].append(edge("n-config", "n-report", "supersedes")) + document["edges"].append(edge("n-config", "n-report", "supersedes")) graph = self.load(document) extra = next(edge for edge in graph.edges if edge.relation == "supersedes") self.assertEqual(extra.kind, query.OTHER_RELATION) @@ -336,7 +406,7 @@ def test_keeps_a_sourceless_stub_traversable_and_uncitable(self) -> None: "id": "n-stub", "label": "Thing", "file_type": "code", "source_file": "", "source_location": "", "origin_file": "example_pkg/config.py", }) - document["links"].append(edge("n-config", "n-stub", "references")) + document["edges"].append(edge("n-config", "n-stub", "references")) graph = self.load(document) self.assertIsNone(graph.nodes["n-stub"].citation) self.assertEqual(len(graph.edges), 5) @@ -347,7 +417,7 @@ def test_drops_non_code_corpora_and_prunes_their_edges(self) -> None: document["nodes"].append( {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} ) - document["links"].append(edge("n-config", "n-doc", "references")) + document["edges"].append(edge("n-config", "n-doc", "references")) graph = self.load(document) self.assertNotIn("n-doc", graph.nodes) self.assertEqual(len(graph.edges), 4) @@ -357,50 +427,37 @@ def test_tolerates_provider_annotations_it_does_not_read(self) -> None: document = graph_document() document["nodes"][0]["metadata"] = {"namespace": "example_pkg", "scope_chain": ["mod"]} document["nodes"][0]["type"] = "namespace" - document["links"][0]["context"] = "call site" + document["edges"][0]["context"] = "call site" self.assertEqual(len(self.load(document).nodes), 5) - def test_refuses_a_document_with_no_provider_nodes_and_links(self) -> None: - for document in ({"nodes": []}, {"links": []}, {"schema": "something.else"}, []): + def test_refuses_a_document_with_no_provider_nodes_and_edges(self) -> None: + for document in ( + {"nodes": []}, {"edges": []}, {"links": []}, {"schema": "something.else"}, [], + ): with self.subTest(document=document): with self.assertRaises(ContextError): self.load(document) def test_refuses_a_graph_built_from_another_commit(self) -> None: - """``built_at_commit`` disagreeing with the generation is a refusal.""" + """``built_at_commit`` disagreeing with the generation is a refusal. + + The stamp is ``to_json``'s, so the raw path never writes one and for a + real ``--no-cluster`` generation this check is vacuous -- the binding + rests on the lifecycle's commit binding and the citation census. It is + still honoured wherever it appears, which is what this covers. + """ with self.assertRaises(ContextError): self.load(graph_document(built_at_commit="c" * 40)) # Agreeing is fine, and is the ordinary case. self.assertEqual(len(self.load(graph_document(built_at_commit="b" * 40)).nodes), 5) - def test_refuses_an_export_that_does_not_preserve_direction(self) -> None: - """An undirected export states an endpoint pair, not a caller and callee. - - The provider's undirected storage canonicalizes endpoint order and its - export's repair leaves no mark a reader can check, so every oriented - answer here -- ``impact``, ``dependency``, and the ``calls`` sentence a - ``symbol`` neighbourhood states -- would be asserting an orientation the - document never established. The refusal names direction, so an operator - reads it as "rebuild directed" rather than as a corrupt graph. - """ - for flag in (False, None, "true", 1): - with self.subTest(directed=flag): - document = graph_document() - if flag is None: - document.pop("directed") - else: - document["directed"] = flag - with self.assertRaises(ContextError) as caught: - self.load(document) - self.assertIn("direction", str(caught.exception)) - def test_refuses_records_missing_the_providers_required_fields(self) -> None: for mutate in ( lambda doc: doc["nodes"][0].pop("label"), lambda doc: doc["nodes"][0].pop("source_file"), lambda doc: doc["nodes"][0].pop("file_type"), - lambda doc: doc["links"][0].pop("relation"), - lambda doc: doc["links"][0].pop("confidence"), + lambda doc: doc["edges"][0].pop("relation"), + lambda doc: doc["edges"][0].pop("confidence"), ): with self.subTest(mutate=mutate): document = graph_document() @@ -411,9 +468,9 @@ def test_refuses_records_missing_the_providers_required_fields(self) -> None: def test_refuses_vocabularies_the_providers_validator_rejects(self) -> None: for mutate in ( lambda doc: doc["nodes"][0].update(file_type="diagram"), - lambda doc: doc["links"][0].update(confidence="GUESSED"), + lambda doc: doc["edges"][0].update(confidence="GUESSED"), # Lowercase is the packet contract's vocabulary, not the provider's. - lambda doc: doc["links"][0].update(confidence="extracted"), + lambda doc: doc["edges"][0].update(confidence="extracted"), ): with self.subTest(mutate=mutate): document = graph_document() @@ -429,7 +486,7 @@ def test_refuses_a_vocabulary_field_that_is_not_text_at_all(self) -> None: callers only ever catch ``ContextError``, so the graph-context path would propagate it instead of reporting the graph unreadable. """ - for field_name, record in (("file_type", "nodes"), ("confidence", "links")): + for field_name, record in (("file_type", "nodes"), ("confidence", "edges")): for value in ([], {}, ["code"], {"value": "EXTRACTED"}, 3, None, True): with self.subTest(field=field_name, value=value): document = graph_document() @@ -461,6 +518,80 @@ def test_refuses_duplicate_node_identifiers(self) -> None: self.load(document) +class NodeLinkFormatTests(unittest.TestCase): + """The other document that can appear under ``graph.json``, read on its own terms. + + ``export.py::to_json`` is the clustered path's writer, and its provenance is + not the extractor's: the endpoints it writes came out of a NetworkX + container that may have been undirected. So it is read, but only when it + says it preserved direction -- and a raw extraction is never held to that, + because its producer writes no such marker. + """ + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_reads_a_directed_node_link_export(self) -> None: + graph = self.load(node_link_document()) + self.assertEqual(len(graph.nodes), 5) + self.assertEqual(len(graph.edges), 4) + + def test_reads_the_renamed_edges_key_of_a_node_link_export(self) -> None: + """NetworkX renamed ``links`` to ``edges``; the pinned validator takes either. + + The marker keys are what identify the format, so the renamed document is + still node-link and is still held to the direction requirement. + """ + document = node_link_document() + document["edges"] = document.pop("links") + self.assertEqual(len(self.load(document).edges), 4) + undirected = node_link_document(directed=False) + undirected["edges"] = undirected.pop("links") + with self.assertRaises(ContextError): + self.load(undirected) + + def test_refuses_a_node_link_export_that_does_not_preserve_direction(self) -> None: + """An undirected export states an endpoint pair, not a caller and callee. + + The provider's undirected storage canonicalizes endpoint order and its + export's repair leaves no mark a reader can check, so every oriented + answer here -- ``impact``, ``dependency``, and the ``calls`` sentence a + ``symbol`` neighbourhood states -- would be asserting an orientation the + document never established. The refusal names direction, so an operator + reads it as "rebuild" rather than as a corrupt graph. + """ + for flag in (False, None, "true", 1): + with self.subTest(directed=flag): + document = node_link_document() + if flag is None: + # Still node-link: ``multigraph``, ``graph`` and ``links`` + # are markers of their own, so dropping one key does not + # make this document pass as a raw extraction. + document.pop("directed") + else: + document["directed"] = flag + with self.assertRaises(ContextError) as caught: + self.load(document) + self.assertIn("direction", str(caught.exception)) + + def test_each_marker_alone_identifies_the_node_link_format(self) -> None: + for marker in query.NODE_LINK_MARKERS: + with self.subTest(marker=marker): + document = graph_document() + document[marker] = {} if marker == "graph" else False + if marker == "links": + document["links"] = document.pop("edges") + with self.assertRaises(ContextError) as caught: + self.load(document) + self.assertIn("direction", str(caught.exception)) + + def test_the_raw_format_is_what_the_reader_reports_for_the_pinned_options(self) -> None: + self.assertEqual(query._graph_format(graph_document()), query.GRAPH_FORMAT_RAW) + self.assertEqual( + query._graph_format(node_link_document()), query.GRAPH_FORMAT_NODE_LINK + ) + + class TraversalTests(GraphWorkspace): def query(self, **overrides) -> query.QueryResult: arguments = {"question": "impact", "target": "parse_config"} @@ -622,7 +753,7 @@ def test_a_citation_past_the_end_of_a_file_is_dropped_not_delivered(self) -> Non def test_confidence_maps_extracted_inferred_and_ambiguous(self) -> None: document = graph_document() - document["links"][2]["confidence"] = "AMBIGUOUS" + document["edges"][2]["confidence"] = "AMBIGUOUS" self.publish(document) outcome = self.context() confidences = {item["confidence"] for item in outcome.packet["documents"]} diff --git a/tests/test_devin_work_orders.py b/tests/test_devin_work_orders.py index ba31b733..2d403d08 100644 --- a/tests/test_devin_work_orders.py +++ b/tests/test_devin_work_orders.py @@ -621,7 +621,10 @@ def assert_paused(self, output, slot, policy="required"): self.assertNotIn("session", output) def test_packet_context_carries_no_identity_and_policy_must_agree_with_the_order(self): - self.assertEqual(tuple(self.context.__dataclass_fields__), ("store", "name", "handle", "policy", "backend")) + self.assertEqual( + tuple(self.context.__dataclass_fields__), + ("store", "name", "handle", "policy", "backend", "revision"), + ) self.assertNotIn(CONTEXT_CANARY, repr(self.context)) self.assertNotIn(self.result["packet_handle"], repr(self.context)) for handle in ("", "not-a-handle", self.result["packet_handle"].upper(), None): @@ -641,6 +644,43 @@ def test_packet_context_carries_no_identity_and_policy_must_agree_with_the_order packet_context(self.fixture.store, "example", self.result["packet_handle"], policy, order=order, backend=self.backend) + def test_the_factory_can_supply_the_consuming_revision(self): + """A repository-kind route needs the consuming commit, so a caller can name it. + + The hosted order has no PR head yet and this process is not the checkout + doing the work, so the revision is the one thing the factory cannot + derive. It reaches ``ContextRequest.revision`` on every render, which is + what a local repository graph is authorized against; leaving it unset + stays the organization-context default. + """ + commit = "c" * 40 + bound = packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=commit) + self.assertEqual(bound.revision, commit) + self.assertNotIn(commit, repr(bound)) + # Unset is still valid, and is what organization context prepares with. + self.assertIsNone(self.context.revision) + for value in ("", "a\nb", "a" * 201, 40): + with self.subTest(revision=value), self.assertRaisesRegex(RemoteError, "invalid_request"): + packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=value) + + def test_the_consuming_revision_reaches_the_context_request(self): + """Whatever the load then decides, the request it decides on names the commit.""" + commit = "d" * 40 + bound = packet_context(self.fixture.store, "example", self.result["packet_handle"], + self.policy, order=self.order, backend=self.backend, revision=commit) + with patch("code_mower.devin_work_orders.load_authorized", wraps=load_authorized) as load: + try: + self.service.run("dispatch", self.order, context=bound, apply=True) + except RemoteError: + pass + self.assertTrue(load.call_args_list) + request = load.call_args_list[0].args[4] + self.assertEqual(request.revision, commit) + self.assertEqual((request.repository, request.work_item), + (self.order.repository, self.order.work_item)) + def test_preview_never_retrieves_context(self): with patch("code_mower.devin_work_orders.load_authorized", wraps=load_authorized) as load: for command in ("dispatch", "clarify", "fix"): From c5639885e50c234aa74c90fa25b2b465988e96ae Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 01:42:44 -0700 Subject: [PATCH 10/33] Graph lifecycle: pass the scan target and derive linked libraries Two prerequisites the adopted extraction path needs before a real build can publish a generation. Both were found by running the unmodified lifecycle against the exact pinned provider, and both are corrections to this repository's launch, not to the pin, the options, or the containment rules. The scan target. The pinned CLI takes the extraction target as the first positional after ``extract``, decides it has one only when that argument does not begin with ``-``, and exits 1 with ``must specify a path to scan or a --postgres DSN`` otherwise. ``subprocess_indexer`` passed the options alone, on the reading that a subcommand which writes beside its sources also discovers them from the working directory, so every real build failed before extraction and failed as the adapter's generic non-zero refusal. ``.`` is now passed between the subcommand and the options: the child's working directory is already the materialized copy, so the relative spelling names that tree and nothing about where it sits on this host. The runtime's linked libraries. CPython's ``_ssl`` extension lives inside the interpreter prefix and is linked against an OpenSSL that does not, and Graphify imports ``ssl`` during start-up even for a code-only extraction, so under a boundary built from prefixes alone the provider aborted inside the loader. The dependency is derived rather than named: every Mach-O image inside the exposure is read for the libraries its own load commands ask dyld to find, each one not already covered is resolved and added, and newly added libraries are read in turn so a transitive dependency is reached without being written down. What is added is the library file, never the directory holding it -- that directory is a package manager's prefix, and its etc and var sit beside it. Loader-relative names are not paths this adds; an absent reference is skipped; every added path goes through the same broad-exposure refusals as any other exposure and is additionally refused unless it is a bounded regular file whose ancestry only this account or root may write. No network denial or filesystem protection is relaxed. The derivation reads Mach-O images, so Linux is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-lifecycle.md | 50 ++- src/code_mower/context_graph_lifecycle.py | 347 ++++++++++++++++++++- tests/test_context_graph_lifecycle.py | 352 +++++++++++++++++++++- 3 files changed, 731 insertions(+), 18 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index 21240dae..eb71342f 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -180,6 +180,35 @@ being read out of a file makes a path no narrower than guessing it would — and an environment that records no base that still exists is refused with an instruction rather than built against whatever runtime is lying around. +A prefix is still not the whole runtime on a host whose interpreter came from a +package manager. CPython's `_ssl` extension lives inside the interpreter's +prefix and is *linked against* an OpenSSL that does not — under Homebrew, +`/opt/homebrew/opt/openssl@3/lib/libssl.3.dylib` and the `libcrypto` beside it — +and Graphify imports `ssl` during start-up even for a code-only extraction. With +only the prefixes exposed those libraries are simply absent, so the provider +aborted inside the loader before it scanned anything. That is a missing runtime +dependency; it is not an argument for giving the child a network or a wider +filesystem, and neither was granted. + +So the dependency is **derived, never named**. Every Mach-O image inside the +exposure is read for the libraries its own load commands ask `dyld` to find, +each one not already covered is resolved and added, and newly added libraries +are read in turn, so `libssl` needing `libcrypto` is reached without either +being written down. What is added is the **library file**, never the directory +holding it: exposing `/opt/homebrew/opt/openssl@3/lib` exposes a package +manager's prefix, and its `etc` and `var` with it. `@rpath`, `@loader_path` and +`@executable_path` names are resolved by `dyld` against the image itself and are +not paths this adds. Every derived path goes through the same broad-exposure +refusals as any other exposure — the filesystem root, the operator's home, the +checkout, an ancestor of either — and is additionally refused unless it is a +regular file within a size bound whose ancestry only this account or root may +write, because a library the provider maps executable inside the boundary is +code. A referenced path that this host does not have installed is skipped: if it +turns out to have been required, the loader fails the build naming the library +it could not find. The derivation reads Mach-O images, so **Linux is unchanged** +— an ELF runtime's libraries are already under the `/lib` and `/usr/lib` +directories the read-only runtime names. + The prefix a particular build ends up with is probed before that build runs, not just the host's mechanism at startup: the readable set of a real build is the provider's install rather than the interpreter paths the host probe uses, @@ -301,13 +330,26 @@ states. It carries no indexed content, no provider output, and no local path. ## How the provider is actually invoked `subprocess_indexer()` builds the argv for the interface the adopt decision -evaluated, not a conventional-looking one: `extract` plus the pinned options, -run with its working directory set to the materialized copy. The evaluated -release takes no `--source`/`--output` pair — `extract` reads the directory it -is run in and writes its state beside those sources, which the clean-room run in +evaluated, not a conventional-looking one: `extract`, the scan target, then the +pinned options, run with its working directory set to the materialized copy. The +evaluated release takes no `--source`/`--output` pair — `extract` writes its +state beside the sources it was pointed at, which the clean-room run in [the evaluation](graphify-evaluation.md) recorded as `extract --code-only --no-cluster --max-workers 4`. +**The scan target is required, and it is positional.** This module used to pass +the options alone, on the reading that a subcommand which writes beside its +sources must also discover them from the working directory. The pinned CLI does +not: it takes the target as the first positional after the subcommand, decides +it has one only when that argument does not begin with `-`, and exits 1 with +`must specify a path to scan or a --postgres DSN` when it does not. Every real +build therefore failed before extraction, and failed as the adapter's generic +non-zero refusal rather than as anything naming the omission. The target passed +is `.`: the child's working directory is already the materialized copy, so the +relative spelling names exactly that tree and names nothing about where it sits +on this host. It goes **between** the subcommand and the options, because the +CLI reads `sys.argv[2]` and nothing later. + **`--code-only` and `--no-cluster` are always passed, whatever the pin says.** They are conditions of the adopt decision, not preferences: a pin that named no options at all would otherwise have launched the provider into clustering and diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index cb77bda7..2390c5c9 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -62,6 +62,7 @@ import signal import socket import stat +import struct import subprocess import sys import tarfile @@ -1592,6 +1593,304 @@ def _refuse_broad_readable(readable: Iterable[str], *, repository: Path) -> None ) +#: Mach-O magics, and the ``struct`` byte order each one means. A Mach-O image +#: declares its own order in its first four bytes; the swapped spellings are how +#: a big-endian image announces itself to a little-endian reader. Both widths +#: are listed with the size of the header that follows, because the load +#: commands this reads begin directly after it. +_MACHO_MAGICS: Mapping[bytes, tuple[str, int]] = { + b"\xcf\xfa\xed\xfe": ("<", 32), # 64-bit, little endian + b"\xce\xfa\xed\xfe": ("<", 28), # 32-bit, little endian + b"\xfe\xed\xfa\xcf": (">", 32), # 64-bit, big endian + b"\xfe\xed\xfa\xce": (">", 28), # 32-bit, big endian +} + +#: A universal ("fat") archive: a big-endian count of architecture records, each +#: naming the offset of a real Mach-O image inside the same file. A python.org +#: interpreter ships these; a Homebrew one does not. +_MACHO_FAT_MAGICS = frozenset({b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca"}) + +#: The load commands that name a library the image will make dyld find. Weak, +#: re-exported and upward links are included: a weak dependency that *is* +#: installed is still opened, and an image that re-exports another still loads +#: it. ``LC_REQ_DYLD`` is the high bit these commands carry. +_MACHO_DYLIB_COMMANDS = frozenset({0x0C, 0x8000_0018, 0x8000_001F, 0x8000_0023}) + +#: How much of an image's load-command block this will read. The block is a +#: header, not the image; anything past this bound is not a Mach-O this module +#: is prepared to reason about. +_MAX_MACHO_COMMAND_BYTES = 4 * 1024 * 1024 + +#: How many architecture slices a universal archive may declare. +_MAX_MACHO_ARCHITECTURES = 32 + +#: How many files the runtime scan will look at, and how many libraries it will +#: add. Both are bounds on somebody else's install, which is the same reason +#: every other foreign file this module reads is bounded: a provider install is +#: not this repository's to trust about its own size. +_MAX_SCANNED_IMAGES = 20_000 +_MAX_LINKED_LIBRARIES = 256 + +#: A shared library that is larger than this is not one; refusing is the honest +#: answer rather than mapping an arbitrary file into the child's view. +_MAX_LINKED_LIBRARY_BYTES = 512 * 1024 * 1024 + +#: File names that are never a Mach-O image, so the scan does not open them. +#: Purely an optimization -- the magic is what decides -- but it is what keeps +#: the walk over a populated ``site-packages`` cheap. +_NOT_MACHO_SUFFIXES = frozenset( + { + ".py", ".pyc", ".pyi", ".pyx", ".txt", ".md", ".rst", ".json", ".toml", + ".yaml", ".yml", ".cfg", ".ini", ".h", ".hpp", ".c", ".cpp", ".html", + ".css", ".js", ".png", ".jpg", ".svg", ".gif", ".pdf", ".zip", ".gz", + ".whl", ".pem", ".crt", ".dist-info", ".egg-info", ".a", ".la", + } +) + +#: Directories the scan does not descend into: build caches and vendored +#: sources, none of which hold an image the child loads. +_NOT_MACHO_DIRECTORIES = frozenset({"__pycache__", ".git", "include", "man", "doc", "docs"}) + + +def _macho_dylib_names(path: Path) -> tuple[str, ...]: + """Every library path a Mach-O image at ``path`` asks dyld to load. + + Read out of the image's own load commands rather than out of ``otool``: + deriving the boundary must not itself depend on a developer tool being + installed, and a parse that reads a bounded header is a smaller thing to + trust than a subprocess. A file that is not a Mach-O -- which is almost + everything under an install prefix -- costs four bytes and returns nothing. + """ + try: + with path.open("rb") as stream: + magic = stream.read(4) + if magic in _MACHO_FAT_MAGICS: + return _macho_fat_dylib_names(stream) + if magic not in _MACHO_MAGICS: + return () + stream.seek(0) + return _macho_slice_dylib_names(stream, 0) + except (OSError, ValueError, struct.error): + # An unreadable or truncated image says nothing about what the runtime + # needs. The build still fails if it was a library the provider loads, + # and it fails as dyld naming the image rather than as this module + # guessing at one. + return () + + +def _macho_fat_dylib_names(stream: io.BufferedReader) -> tuple[str, ...]: + """The union over a universal archive's slices. + + The union rather than the slice matching this process: the child is the + provider's interpreter, whose architecture is not necessarily this one, and + every slice's dependencies are paths on the same host. + """ + count = struct.unpack(">I", stream.read(4))[0] + if count > _MAX_MACHO_ARCHITECTURES: + return () + offsets = [] + for _ in range(count): + record = stream.read(20) + if len(record) != 20: + return () + # cputype, cpusubtype, offset, size, align + offsets.append(struct.unpack(">5I", record)[2]) + names: dict[str, None] = {} + for offset in offsets: + for name in _macho_slice_dylib_names(stream, offset): + names.setdefault(name, None) + return tuple(names) + + +def _macho_slice_dylib_names(stream: io.BufferedReader, offset: int) -> tuple[str, ...]: + """The ``LC_LOAD_DYLIB`` family of one Mach-O image beginning at ``offset``.""" + stream.seek(offset) + magic = stream.read(4) + order_and_header = _MACHO_MAGICS.get(magic) + if order_and_header is None: + return () + order, header_size = order_and_header + header = stream.read(header_size - 4) + if len(header) != header_size - 4: + return () + # cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags[, reserved] + ncmds, sizeofcmds = struct.unpack(f"{order}6I", header[:24])[3:5] + if sizeofcmds > _MAX_MACHO_COMMAND_BYTES: + return () + block = stream.read(sizeofcmds) + names: dict[str, None] = {} + position = 0 + for _ in range(ncmds): + if position + 8 > len(block): + break + command, size = struct.unpack_from(f"{order}2I", block, position) + if size < 8 or position + size > len(block): + break + if command in _MACHO_DYLIB_COMMANDS and size >= 24: + name_offset = struct.unpack_from(f"{order}I", block, position + 8)[0] + if 8 <= name_offset < size: + raw = block[position + name_offset : position + size] + name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") + if name: + names.setdefault(name, None) + position += size + return tuple(names) + + +def _scan_images(root: Path, *, budget: list[int]) -> Iterator[Path]: + """Regular files under ``root`` that could be Mach-O images, within a budget. + + ``budget`` is shared across every root of one derivation, so the cost is a + property of the whole runtime rather than of each prefix in it. Symlinks are + not followed during the walk: an install that links a directory elsewhere is + reached through whatever named it, and following would let one link turn a + narrow prefix into an unbounded traversal. + """ + for parent, directories, files in os.walk(root, followlinks=False): + directories[:] = [name for name in directories if name not in _NOT_MACHO_DIRECTORIES] + for name in files: + if budget[0] <= 0: + return + if any(name.endswith(suffix) for suffix in _NOT_MACHO_SUFFIXES): + continue + budget[0] -= 1 + yield Path(parent) / name + + +def _trusted_library(path: Path) -> bool: + """Could only a trusted account have put this library where the child reads it? + + The same question :func:`_trusted_launcher` asks of a sandbox launcher, and + for the same reason: a library the provider maps executable inside the + boundary is code, and a file -- or a directory above it -- that some other + account may write is a file somebody else chooses the contents of. Asked of + the *resolved* path, so a link's own spelling is not what is trusted. + """ + trusted = {0, os.geteuid()} + for current in (path, *path.parents): + try: + entry = os.lstat(current) + except OSError: + return False + writable = bool(entry.st_mode & (stat.S_IWGRP | stat.S_IWOTH)) + if writable and stat.S_ISDIR(entry.st_mode) and entry.st_mode & stat.S_ISVTX: + # A sticky shared directory -- ``/tmp`` and the per-user temporary + # directories under it. Another account may create its own entries + # there and may not touch this one, which is the whole point of the + # bit, so the ancestry it provides is not an account boundary this + # has to refuse. The file itself is still held to the rule. + continue + if entry.st_uid not in trusted or writable: + return False + return True + + +def _linked_runtime_libraries( + roots: Sequence[Path], *, covered: Sequence[Path], repository: Path +) -> tuple[str, ...]: + """Shared libraries the exposed runtime links to from outside the exposure. + + A pinned provider's install and the base interpreter it was created from are + exposed as prefixes, and that was taken to be the whole runtime. It is not, + on a host whose interpreter was installed by a package manager: CPython's + ``_ssl`` extension is linked against an OpenSSL that lives under the + manager's own prefix, not under the interpreter's, and Graphify imports + ``ssl`` during start-up even for a code-only extraction. Under the + filesystem boundary that library is simply absent, so the provider aborted + at import time -- which is a missing runtime dependency, not an argument for + giving the child a network or a wider filesystem. + + So the dependency is *derived* rather than named. Every Mach-O image inside + the exposure is read for the libraries it asks dyld to load, and each one + that is not already covered is resolved and added as a single file. Newly + added libraries are read in turn, so a transitive dependency -- ``libssl`` + needing ``libcrypto`` -- is reached without either being written down here. + + What is added is the library file, never the directory holding it: exposing + ``/opt/homebrew/opt/openssl@3/lib`` is a package manager's prefix, and + exposing the manager's ``etc`` or ``var`` beside it is the operator data + this boundary exists to withhold. Every added path is put through the same + ownership and broad-exposure refusals as any other exposure, and a path that + is not a bounded regular file is refused rather than exposed on the strength + of an image having named it. + + Linux is unchanged: an ELF runtime's libraries live under the ``/lib`` and + ``/usr/lib`` directories the read-only runtime already names, and this + derivation reads Mach-O images, of which such a host has none. + """ + if sys.platform != "darwin": + return () + system = tuple(Path(os.path.realpath(path)) for path in _SYSTEM_READ_PATHS) + boundaries = [*system, *(Path(os.path.realpath(root)) for root in covered)] + added: dict[str, None] = {} + pending = [Path(os.path.realpath(root)) for root in roots] + budget = [_MAX_SCANNED_IMAGES] + while pending: + current = pending.pop(0) + images = _scan_images(current, budget=budget) if current.is_dir() else iter((current,)) + for image in images: + for name in _macho_dylib_names(image): + if not name.startswith("/"): + # ``@rpath``, ``@loader_path`` and ``@executable_path`` are + # resolved by dyld against the image itself, so they name + # something inside the exposure already. + continue + referenced = Path(name) + resolved = Path(os.path.realpath(referenced)) + if any(_under(resolved, boundary) for boundary in boundaries): + continue + if str(resolved) in added: + continue + if not resolved.exists(): + # A weak dependency the host does not have installed. If it + # was a required one the provider fails at launch, as dyld + # naming the library it could not find. + continue + _refuse_linked_library(resolved, repository=repository) + if len(added) >= _MAX_LINKED_LIBRARIES: + raise ContextError( + "the local graph provider's runtime links to more shared libraries " + "outside its install than this boundary is willing to expose; " + "no generation was published" + ) + added[str(resolved)] = None + if str(referenced) != str(resolved): + # Both spellings, for the same reason the executable has + # two: a bind-mount boundary names a literal destination, + # and dyld opens the path the image wrote down. + added.setdefault(str(referenced), None) + pending.append(resolved) + return tuple(added) + + +def _refuse_linked_library(resolved: Path, *, repository: Path) -> None: + """Refuse a derived dependency that is not a library this may expose.""" + _refuse_broad_exposure(resolved, repository=repository) + try: + info = os.lstat(resolved) + except OSError: # pragma: no cover - the caller has just seen it exist + raise ContextError( + "a shared library the local graph provider's runtime links to could not be " + "read while deriving the containment boundary; no generation was published" + ) from None + if not stat.S_ISREG(info.st_mode): + raise ContextError( + "the local graph provider's runtime links to something that is not a regular " + "file; refusing to expose it to the sandbox" + ) + if info.st_size > _MAX_LINKED_LIBRARY_BYTES: + raise ContextError( + "a shared library the local graph provider's runtime links to is implausibly " + "large for one; refusing to expose it to the sandbox" + ) + if not _trusted_library(resolved): + raise ContextError( + "a shared library the local graph provider's runtime links to is writable by " + "an account other than yours or root, so what the provider would load inside " + "the sandbox is not what this host installed; no generation was published" + ) + + def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: """The install the pinned provider needs to be readable, and nothing beside it. @@ -1640,7 +1939,16 @@ def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: # out of *this* environment's own ``pyvenv.cfg`` rather than taken from the # interpreter Code Mower happens to be running under, which is a different # installation whenever the provider was pinned with a different Python. - return (*spellings, str(root), *_provider_base_prefixes(root, repository=repository)) + prefixes = (str(root), *_provider_base_prefixes(root, repository=repository)) + # Last, and derived from the prefixes rather than added to them: a prefix is + # not the whole runtime on a host whose interpreter links against libraries + # a package manager keeps somewhere else. + libraries = _linked_runtime_libraries( + [Path(prefix) for prefix in prefixes], + covered=[Path(prefix) for prefix in prefixes], + repository=repository, + ) + return (*spellings, *prefixes, *libraries) #: Where an installed distribution records its own identity (PEP 376). The @@ -1869,12 +2177,32 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: #: The subcommand the evaluated release exposes, recorded in #: ``docs/graphify-evaluation.md``: the clean-room run indexed with #: ``extract --code-only --no-cluster --max-workers 4``. There is no -#: ``--source``/``--output`` pair to hand it; ``extract`` reads the directory -#: it is run in and writes its state beside those sources, which is why the -#: child's working directory is the materialized copy and why the adapter -#: collects an artifact afterwards rather than naming one up front. +#: ``--source``/``--output`` pair to hand it; ``extract`` writes its state +#: beside the sources it was pointed at, which is why the child's working +#: directory is the materialized copy and why the adapter collects an artifact +#: afterwards rather than naming one up front. _PROVIDER_EXTRACT = "extract" +#: The scan target, which the pinned CLI requires and does not default. +#: +#: This module used to launch ``extract`` with the options alone, on the reading +#: that a subcommand which writes beside its sources must also discover them +#: from the working directory. The pinned CLI does not: it takes the target as +#: the first positional after the subcommand, decides it has one only when that +#: argument does not begin with ``-``, and exits 1 with ``must specify a path to +#: scan or a --postgres DSN`` when it does not. Every real build therefore +#: failed before extraction, and the failure arrived as the generic non-zero +#: refusal rather than as anything naming the omission. +#: +#: ``.`` rather than the source root's absolute path: the child's working +#: directory is already the materialized copy, so the relative spelling names +#: exactly the tree this build means and names nothing about where that tree +#: sits on the host. It must be passed *between* the subcommand and the options +#: -- the CLI reads ``sys.argv[2]`` and nothing later -- and it is a path, so a +#: bare ``.`` can never be mistaken for a flag the way a caller-supplied string +#: could. +_PROVIDER_SCAN_TARGET = "." + #: The provider's own record of what it processed. Completeness is read from #: here, never inferred from an exit status: the clean-room run recorded 54 #: manifest entries requeued by a repeat that exited zero in 1.63 s. @@ -2302,9 +2630,10 @@ def subprocess_indexer( callable that would run whatever a later request's pin happened to name. The argv is the interface the adopt decision evaluated, not a guess at a - conventional one: ``extract`` with the required restrictions and then the - pinned options, in the materialized copy. Everything the provider leaves - behind is then collected and classified from its own report. + conventional one: ``extract``, the scan target the pinned CLI requires, the + required restrictions and then the pinned options, in the materialized copy. + Everything the provider leaves behind is then collected and classified from + its own report. """ command = _resolved_executable(executable) if containment_mechanism() is None: @@ -2351,7 +2680,7 @@ def run(request: IndexRequest) -> IndexResult: options = _extraction_options(request.pin.options) try: returncode = _run_contained( - [*sandbox, command, _PROVIDER_EXTRACT, *options], + [*sandbox, command, _PROVIDER_EXTRACT, _PROVIDER_SCAN_TARGET, *options], environment=request.environment, cwd=str(request.source_root), timeout=EXTRACTION_TIMEOUT_SECONDS, diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index a29a8882..2110a48e 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -25,6 +25,7 @@ import os import socket import stat +import struct import subprocess import sys import tarfile @@ -1176,13 +1177,68 @@ def test_the_provider_leads_its_own_process_group(self) -> None: def test_the_provider_is_invoked_through_its_documented_extract_interface(self) -> None: # The interface the adopt decision evaluated, recorded in - # docs/graphify-evaluation.md as ``extract`` plus options. An - # ``index --source ... --output ...`` shape would be a different CLI. + # docs/graphify-evaluation.md as ``extract`` over a scan target plus + # options. An ``index --source ... --output ...`` shape would be a + # different CLI. argv = self.launched_argv("graphify") - self.assertEqual(argv[3:], ["extract", *PIN.options]) + self.assertEqual(argv[3:], ["extract", ".", *PIN.options]) self.assertNotIn("--source", argv) self.assertNotIn("--output", argv) + def test_the_scan_target_is_passed_where_the_pinned_cli_reads_it(self) -> None: + """The pinned CLI requires a target and reads it from one position only. + + It takes the target as the first positional after the subcommand, + decides it has one only when that argument does not begin with ``-``, + and exits 1 with ``must specify a path to scan or a --postgres DSN`` + otherwise. The launch used to pass the options alone, so every real + build failed before extraction -- and failed as the adapter's generic + non-zero refusal, which names nothing about the omission. + + ``.`` rather than an absolute path: the child's working directory is + already the materialized copy, so the relative spelling names that tree + and nothing about where it sits on this host. + """ + argv = self.launched_argv("graphify") + subcommand = argv.index("extract") + self.assertEqual(argv[subcommand + 1], ".") + self.assertFalse(argv[subcommand + 1].startswith("-")) + # Before the options, not after them: an argument that follows a flag is + # read as that flag's value or skipped, and either way the CLI still + # sees no path. + for option in PIN.options: + self.assertLess(subcommand + 1, argv.index(option)) + + def test_the_pinned_cli_would_accept_this_argv(self) -> None: + """Read the requirement off the staged pinned source, not off a memory. + + The reference under ``.build/graphify-914-reference`` is the CLI this + pin actually installs. If a later pin stops requiring a positional + target, or starts reading it from somewhere other than ``sys.argv[2]``, + this test is what says so rather than a real build failing opaquely. + """ + reference = ( + Path(__file__).resolve().parent.parent + / ".build" + / "graphify-914-reference" + / "graphify-cli-pinned.py" + ) + if not reference.is_file(): + # The reference is a local read-only staging of the pinned provider's + # own source, excluded from the repository rather than vendored into + # it, so this check runs where it is staged and skips where it is not. + self.skipTest("pinned CLI reference is not staged in this checkout") + source = reference.read_text(encoding="utf-8", errors="replace") + self.assertIn("error: must specify a path to scan or a --postgres DSN", source) + # The target is ``sys.argv[2]`` and a leading dash means "no path". + self.assertIn('if sys.argv[2].startswith("-"):', source) + argv = self.launched_argv("graphify") + provider = argv.index(str(self.provider)) + # ``sys.argv`` inside the child is the provider and everything after it, + # so ``sys.argv[2]`` is the second argument past the executable. + self.assertEqual(argv[provider + 1], "extract") + self.assertEqual(argv[provider + 2], ".") + def test_extraction_is_restricted_to_code_and_never_clusters(self) -> None: """The adoption conditions are enforced at the launch, not assumed. @@ -1193,7 +1249,7 @@ def test_extraction_is_restricted_to_code_and_never_clusters(self) -> None: """ bare = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) argv, _ = self.run_indexer("graphify", pin=bare) - self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + self.assertEqual(argv[3:], ["extract", ".", "--code-only", "--no-cluster"]) def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: # The pin normalizes its own options, so this reaches past the @@ -1201,7 +1257,7 @@ def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: stripped = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) object.__setattr__(stripped, "options", ()) argv, _ = self.run_indexer("graphify", pin=stripped) - self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + self.assertEqual(argv[3:], ["extract", ".", "--code-only", "--no-cluster"]) def test_the_collected_artifact_holds_the_state_the_provider_wrote(self) -> None: _, result = self.run_indexer("graphify") @@ -1849,6 +1905,286 @@ def test_a_readable_set_reaching_the_home_directory_is_refused(self) -> None: ) +def macho(path: Path, *dependencies: str) -> Path: + """A real 64-bit Mach-O header whose load commands name ``dependencies``. + + A header, not a whole image: the derivation reads ``LC_LOAD_DYLIB`` out of + the load-command block and nothing else, so a file with a real magic, a real + command count and real commands exercises exactly the parse under test + without needing a compiler on the machine running these tests. + """ + commands = b"" + for name in dependencies: + raw = name.encode("utf-8") + b"\0" + raw += b"\0" * ((-len(raw)) % 8) + # cmd=LC_LOAD_DYLIB, cmdsize, name offset, timestamp, versions. + commands += struct.pack("<6I", 0x0C, 24 + len(raw), 24, 0, 0, 0) + raw + # magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags, reserved + header = struct.pack( + "<8I", 0xFEEDFACF, 0x0100_000C, 0, 6, len(dependencies), len(commands), 0, 0 + ) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + commands) + return path + + +class LinkedRuntimeLibraryTests(ProviderExposureFixture): + """Shared libraries the exposed runtime links to from outside the exposure. + + A pinned provider's environment and the base interpreter it was created from + were exposed as prefixes, and that was taken to be the whole runtime. On a + host whose interpreter came from a package manager it is not: CPython's + ``_ssl`` extension is linked against an OpenSSL under the *manager's* prefix, + and Graphify imports ``ssl`` during start-up even for a code-only + extraction, so under this boundary the provider aborted before it scanned + anything. + + These hold the repair to the shape it has to have: the dependency is derived + from the images themselves rather than named here, what is exposed is a + library file rather than the manager's prefix, every added path goes through + the same refusals as any other exposure, and Linux is untouched. + """ + + def derive(self, *prefixes: Path) -> tuple[str, ...]: + with mock.patch.object(sys, "platform", "darwin"): + return lifecycle._linked_runtime_libraries( + list(prefixes), covered=list(prefixes), repository=self.repository + ) + + def library(self, path: Path, *dependencies: str) -> Path: + return macho(path, *dependencies) + + def test_a_library_outside_the_exposure_is_added_as_a_file(self) -> None: + brew = self.root / "brew" / "Cellar" / "openssl@3" / "3.6.3" / "lib" + libssl = self.library(brew / "libssl.3.dylib") + prefix = self.root / "python" + self.library(prefix / "lib-dynload" / "_ssl.so", str(libssl)) + derived = self.derive(prefix) + self.assertIn(str(libssl), derived) + # The file, never the directory holding it: exposing that directory is + # exposing a package manager's prefix, and its ``etc`` and ``var`` with + # it. + self.assertNotIn(str(brew), derived) + self.assertNotIn(str(brew.parent), derived) + self.assertNotIn(str(self.root / "brew"), derived) + + def test_a_transitive_dependency_is_reached(self) -> None: + """``libssl`` needs ``libcrypto``, and neither is written down here.""" + brew = self.root / "brew" / "lib" + libcrypto = self.library(brew / "libcrypto.3.dylib") + libssl = self.library(brew / "libssl.3.dylib", str(libcrypto)) + prefix = self.root / "python" + self.library(prefix / "lib-dynload" / "_ssl.so", str(libssl)) + derived = self.derive(prefix) + self.assertIn(str(libssl), derived) + self.assertIn(str(libcrypto), derived) + + def test_a_link_is_resolved_and_both_spellings_are_kept(self) -> None: + """A manager's stable ``opt`` name is a link into its versioned cellar. + + dyld opens the path the image wrote down; a bind-mount boundary names a + literal destination in an otherwise empty root. Neither spelling is the + other, so both are exposed -- and the file that is *validated* is the + resolved one, because a link's own spelling is not what gets read. + """ + cellar = self.library(self.root / "brew" / "Cellar" / "o" / "3" / "lib" / "libssl.dylib") + stable = self.root / "brew" / "opt" / "openssl@3" + stable.parent.mkdir(parents=True, exist_ok=True) + stable.symlink_to(cellar.parent.parent) + referenced = stable / "lib" / "libssl.dylib" + prefix = self.root / "python" + self.library(prefix / "_ssl.so", str(referenced)) + derived = self.derive(prefix) + self.assertIn(str(cellar), derived) + self.assertIn(str(referenced), derived) + + def test_a_dependency_already_inside_the_exposure_is_not_added(self) -> None: + prefix = self.root / "python" + inside = self.library(prefix / "lib" / "libpython.dylib") + self.library(prefix / "lib-dynload" / "_x.so", str(inside)) + self.assertEqual(self.derive(prefix), ()) + + def test_a_dependency_the_read_only_runtime_already_covers_is_not_added(self) -> None: + """``/usr/lib/libSystem.B.dylib`` is in every child's view already.""" + prefix = self.root / "python" + self.library(prefix / "_x.so", "/usr/lib/libSystem.B.dylib") + self.assertEqual(self.derive(prefix), ()) + + def test_loader_relative_names_are_not_treated_as_paths(self) -> None: + """``@rpath`` and friends are resolved by dyld against the image itself.""" + prefix = self.root / "python" + self.library( + prefix / "_x.so", + "@rpath/libfoo.dylib", + "@loader_path/../libbar.dylib", + "@executable_path/libbaz.dylib", + ) + self.assertEqual(self.derive(prefix), ()) + + def test_an_absent_dependency_is_skipped_rather_than_refused(self) -> None: + """A weak link to something this host never installed. + + Skipped, not refused: if it turns out to have been required, the + provider fails at launch as dyld naming the library it could not find, + which is a better answer than this module refusing a build over a link + that is never opened. + """ + prefix = self.root / "python" + self.library(prefix / "_x.so", str(self.root / "brew" / "lib" / "libgone.dylib")) + self.assertEqual(self.derive(prefix), ()) + + def test_a_dependency_inside_the_checkout_is_refused(self) -> None: + """The live working tree is what the materialized copy exists to replace.""" + inside = self.library(self.repository / "vendor" / "libevil.dylib") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(inside)) + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_a_dependency_that_is_the_home_directory_is_refused(self) -> None: + home = self.root / "home" + home.mkdir() + prefix = self.root / "python" + self.library(prefix / "_x.so", str(home)) + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_a_dependency_that_is_not_a_regular_file_is_refused(self) -> None: + directory = self.root / "brew" / "lib" + directory.mkdir(parents=True) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(directory)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a regular file", str(raised.exception)) + + def test_an_implausibly_large_dependency_is_refused(self) -> None: + big = self.root / "brew" / "lib" / "libhuge.dylib" + big.parent.mkdir(parents=True) + big.write_bytes(b"") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(big)) + with mock.patch.object(lifecycle, "_MAX_LINKED_LIBRARY_BYTES", -1): + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("implausibly large", str(raised.exception)) + + def test_a_dependency_another_account_may_rewrite_is_refused(self) -> None: + """What the provider maps executable inside the boundary is code. + + A library some other account may write is a library somebody else + chooses the contents of, and the sandbox would then be confining the + provider to a runtime this host did not install. + """ + loose = self.library(self.root / "brew" / "lib" / "libloose.dylib") + loose.chmod(0o666) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(loose)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("writable by", str(raised.exception)) + + def test_more_libraries_than_the_bound_are_refused(self) -> None: + brew = self.root / "brew" / "lib" + names = [str(self.library(brew / f"lib{index}.dylib")) for index in range(4)] + prefix = self.root / "python" + self.library(prefix / "_x.so", *names) + with mock.patch.object(lifecycle, "_MAX_LINKED_LIBRARIES", 2): + with self.assertRaises(ContextError): + self.derive(prefix) + + def test_linux_derives_nothing(self) -> None: + """An ELF runtime's libraries are already under the read-only runtime. + + The derivation reads Mach-O images, of which such a host has none, and + the previous behaviour there is the whole behaviour. + """ + brew = self.root / "brew" / "lib" + libssl = self.library(brew / "libssl.so") + prefix = self.root / "python" + self.library(prefix / "_ssl.so", str(libssl)) + with mock.patch.object(sys, "platform", "linux"): + self.assertEqual( + lifecycle._linked_runtime_libraries( + [prefix], covered=[prefix], repository=self.repository + ), + (), + ) + + def test_a_file_that_is_not_an_image_reads_as_no_dependencies(self) -> None: + prefix = self.root / "python" + prefix.mkdir() + (prefix / "notes").write_bytes(b"not a mach-o at all") + (prefix / "truncated.dylib").write_bytes(b"\xcf\xfa\xed\xfe") + self.assertEqual(self.derive(prefix), ()) + + def test_a_universal_archive_is_read_slice_by_slice(self) -> None: + """python.org ships fat binaries; the union of the slices is the answer.""" + brew = self.root / "brew" / "lib" + first = self.library(brew / "libone.dylib") + second = self.library(brew / "libtwo.dylib") + slices = [ + macho(self.root / "slice-one", str(first)).read_bytes(), + macho(self.root / "slice-two", str(second)).read_bytes(), + ] + header = struct.pack(">2I", 0xCAFEBABE, 2) + offset = len(header) + 40 + body = b"" + arches = b"" + for payload in slices: + arches += struct.pack(">5I", 0x0100_000C, 0, offset + len(body), len(payload), 0) + body += payload + prefix = self.root / "python" + prefix.mkdir(exist_ok=True) + (prefix / "fat.dylib").write_bytes(header + arches + body) + derived = self.derive(prefix) + self.assertIn(str(first), derived) + self.assertIn(str(second), derived) + + def test_the_provider_exposure_carries_the_derived_libraries(self) -> None: + """The whole point: what ``subprocess_indexer`` confines the child to.""" + brew = self.root / "brew" / "lib" + libssl = self.library(brew / "libssl.3.dylib") + environment = self.root / "venv" + provider = self.script(environment / "bin" / "graphify", venv=True) + self.library(environment / "lib" / "_ssl.so", str(libssl)) + with mock.patch.object(sys, "platform", "darwin"): + exposed = self.exposure(provider) + self.assertIn(str(libssl), exposed) + self.assertIn(str(environment), exposed) + self.assertNotIn(str(brew), exposed) + + @unittest.skipUnless(sys.platform == "darwin", "Mach-O linkage is a macOS question") + def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: + """Against this host's actual interpreter, not a fixture. + + The fixtures above prove the parse and the refusals; only this proves + the thing the blocker was about -- that the libraries CPython's ``_ssl`` + really links against end up inside the boundary. Every absolute + dependency that extension names must be either under the read-only + runtime every child gets or in the derived set. + """ + import _ssl # noqa: PLC0415 - the point is this host's real extension + + extension = Path(getattr(_ssl, "__file__", "") or "") + if not extension.is_file(): # pragma: no cover - a statically linked build + self.skipTest("this interpreter's _ssl is not a separate extension module") + prefix = Path(sys.base_prefix) + derived = lifecycle._linked_runtime_libraries( + [prefix], covered=[prefix], repository=self.repository + ) + system = [Path(os.path.realpath(path)) for path in lifecycle._SYSTEM_READ_PATHS] + covered = [*system, Path(os.path.realpath(prefix))] + for name in lifecycle._macho_dylib_names(extension): + if not name.startswith("/"): + continue + resolved = Path(os.path.realpath(name)) + if any(lifecycle._under(resolved, root) for root in covered): + continue + self.assertIn(str(resolved), derived, f"{name} is outside the provider's boundary") + + class ProviderIdentityTests(TemporaryWorkspace): """The pin is checked against the install, before the install is run. @@ -3043,6 +3379,12 @@ def indexer_script(self, *, complete: bool = True, **installed) -> Path: body=( "#!/bin/sh\n" '[ "$1" = "extract" ] || exit 64\n' + # The pinned CLI requires a scan target here and exits 1 without + # one, so the stand-in refuses the same argv the real provider + # refuses rather than accepting a launch that could never work. + '[ -n "$2" ] || exit 65\n' + 'case "$2" in -*) exit 65 ;; esac\n' + '[ -d "$2" ] || exit 65\n' "mkdir -p .graphify\n" "printf graph-bytes > .graphify/graph.bin\n" 'printf \'{"complete": %s, "code_files": 1, "requeued": 0}\' ' From 3960d9815decabbb34801ea3c03d3ce4cb1aa568 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 08:34:25 -0700 Subject: [PATCH 11/33] Graph lifecycle: collect the pin's real output and prove coverage Read off the hash-verified graphifyy 0.9.58 wheel rather than assumed. Output root. `graphify/paths.py` builds every output path from `GRAPHIFY_OUT`, whose default is the literal `graphify-out`, so an unmodified `extract` in the materialized copy leaves `graphify-out/graph.json` and `graphify-out/manifest.json` beneath the scan target. The collector only accepted `.graphify`/`.graph`, so it found nothing after a real run. It now resolves exactly `graphify-out` beneath the copy -- never a caller path and never the environment override that constant reads -- requires a real directory that is not a symlink, requires `graph.json` as a regular file, and refuses a second provider root beside it rather than guessing which one a generation comes from. `graphify-out` joins the pre-existing-state refusal and the excluded roots, so a tracked one is neither materialized, censused, nor citable, under the policy `.graphify` and `.graph` already had. Completeness. `save_manifest` writes a flat relative-path map to `{mtime, seen, ast_hash, semantic_hash}` -- not a completion flag or an indexed count, which is what this adapter was reading for. Completeness is now coverage: the denominator is the immutable materialized census narrowed to the pin's own `CODE_EXTENSIONS`, and a file is processed only when its row carries a well-formed `_md5_file` digest. Inputs outside that set are deterministically not code to this pin and are skipped, not missing. The pin blanks the hash through `clear_ast` on an extractor error or an anomalous zero-node extract, so the clean-room run's 54 requeued entries are blank rows and stay partial. A missing row, a malformed row, an unreadable manifest and a request carrying no census all stay partial too. Nothing upgrades a run: exit status, a non-empty graph, and the raw `extracted_sources` all describe what was dispatched, failures included. The real-SSL test no longer assumes every host runtime can be admitted. A shared package-manager prefix with group-writable ancestry is refused on purpose; the test now asks this host independently whether the runtime or one of `_ssl`'s dependencies really is untrusted and accepts the refusal only then. A refusal over a trusted runtime fails, and any other exception is not caught. Not executed: this builder shell has no usable interpreter. Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_graph.py | 2 +- src/code_mower/context_graph_lifecycle.py | 331 ++++++++++++++-------- tests/test_context_graph_lifecycle.py | 306 +++++++++++++++----- 3 files changed, 446 insertions(+), 193 deletions(-) diff --git a/src/code_mower/context_graph.py b/src/code_mower/context_graph.py index 07f5390e..ce1f124c 100644 --- a/src/code_mower/context_graph.py +++ b/src/code_mower/context_graph.py @@ -31,7 +31,7 @@ #: these means the graph escaped the immutable checkout it was asked to index. #: Compared case-folded: on a case-insensitive filesystem (APFS and NTFS by #: default) ``.GIT/config`` names the same directory as ``.git/config``. -_EXCLUDED_ROOTS = frozenset({".git", ".graph", ".graphify", ".code-mower"}) +_EXCLUDED_ROOTS = frozenset({".git", ".graph", ".graphify", "graphify-out", ".code-mower"}) def _names_private_state(parts: Iterable[str]) -> bool: diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index 2390c5c9..a5a0c929 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -68,10 +68,11 @@ import tarfile import tempfile import time +import unicodedata import uuid from dataclasses import dataclass from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence from .context_contract import ContextError, _identifier, _text, _timestamp @@ -114,15 +115,32 @@ _REGULAR_MODES = frozenset({"100644", "100755"}) _SKIPPED_MODES = {"120000": "symlink", "160000": "submodule"} -#: Where the provider keeps its own index state. Both names are on the -#: excluded-roots list in ``context_graph``, and a repository is free to track -#: either of them -- a committed ``.graph/`` is somebody else's graph, or an -#: earlier incremental cache of this one. Neither may be materialized: the +#: Where the pinned provider actually writes. Read off ``graphify/paths.py`` at +#: the evaluated pin rather than assumed: ``GRAPHIFY_OUT`` defaults to the +#: literal ``graphify-out`` and every output path is built from it, so an +#: unmodified ``extract`` run in the materialized copy leaves +#: ``graphify-out/graph.json`` and ``graphify-out/manifest.json`` beneath the +#: scan target. The environment override that constant reads is deliberately +#: *not* honoured here: the child is given a fixed environment, and an output +#: root this adapter did not choose is a root it cannot bound to the copy. +_PROVIDER_OUTPUT_DIRECTORY = "graphify-out" + +#: The document the ``--no-cluster`` branch dumps, and the provider's own record +#: of which inputs it processed. Exactly these two names are read; a run that +#: leaves something else has not produced the evidence this adapter classifies. +_PROVIDER_GRAPH_NAME = "graph.json" +_PROVIDER_MANIFEST_NAME = "manifest.json" + +#: Where the provider keeps its own index state or output. All three names are +#: on the excluded-roots list in ``context_graph``, and a repository is free to +#: track any of them -- a committed ``.graph/`` is somebody else's graph, a +#: committed ``graphify-out/`` is an earlier build's published output, and +#: ``.graphify/`` is an incremental cache. None may be materialized: the #: provider would then resume from a cache built over content this build never #: saw, and the adapter would collect tracked repository bytes as if the #: provider had just produced them, binding stale contents to a fresh commit. #: Matched at any depth and case-folded, for the same reasons ``.git`` is. -_PROVIDER_STATE_DIRECTORIES = (".graphify", ".graph") +_PROVIDER_STATE_DIRECTORIES = (_PROVIDER_OUTPUT_DIRECTORY, ".graphify", ".graph") _PROVIDER_STATE_ROOTS = frozenset(name.casefold() for name in _PROVIDER_STATE_DIRECTORIES) #: The evidence contract's excluded roots, bound rather than copied. One module @@ -1330,6 +1348,13 @@ class IndexRequest: #: directory the environment points at but the sandbox does not expose is a #: provider that cannot start. writable: tuple[Path, ...] = () + #: The census the copy was materialized from -- the denominator completeness + #: is measured against. Carried on the request rather than re-read from the + #: copy after the run, because by then the provider has written into that + #: tree: the question is what this build *gave* the provider, and only the + #: census is immutable evidence of that. A request without one cannot be + #: classified as complete, which is the safe direction. + census: TrackedCensus | None = None @dataclass(frozen=True) @@ -2203,31 +2228,39 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: #: could. _PROVIDER_SCAN_TARGET = "." -#: The provider's own record of what it processed. Completeness is read from -#: here, never inferred from an exit status: the clean-room run recorded 54 -#: manifest entries requeued by a repeat that exited zero in 1.63 s. -_PROVIDER_REPORT_NAMES = ("manifest.json", "index.json", "report.json") - -#: Counters whose presence above zero means the provider did not finish. Any -#: one of them, not all: a report that admits requeued entries is a partial -#: build however healthy the rest of it looks. -_INCOMPLETE_COUNTERS = ("requeued", "pending", "failed", "errors", "incomplete") - -#: Where the provider reports how many files it actually indexed. -_INDEXED_COUNTERS = ("indexed_files", "code_files", "files", "entries") - -#: An affirmative claim that the run finished, in either shape a report can -#: carry one. Nothing else counts: an empty object, or one whose schema this -#: adapter does not recognize, says nothing about completion and is therefore -#: not evidence of it. -_COMPLETION_FLAGS = ("complete", "completed", "finished") -#: Narrow on purpose: a field that names the run's state, not one that might -#: carry a path or a message, so an unrecognized value here is a real -#: non-completion rather than an adapter that read the wrong field. -_COMPLETION_STATUS_FIELDS = ("status", "state") -_COMPLETION_STATUS_VALUES = frozenset( - {"complete", "completed", "success", "succeeded", "ok", "finished", "done"} -) +#: The extensions the pinned provider's own ``detect.CODE_EXTENSIONS`` treats as +#: code, transcribed from the hash-verified 0.9.58 wheel. This is the +#: denominator's definition and it has to be the provider's, not a plausible +#: one: a build is complete when every input the provider itself would dispatch +#: was processed, and holding it to every tracked documentation file instead +#: would make a correct run permanently partial. Nothing outside this set is +#: counted against the run -- those inputs are deterministically not code to +#: this pin, so they are skipped rather than missing. +_PROVIDER_CODE_EXTENSIONS = frozenset({ + ".py", ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs", ".ejs", + ".ets", ".go", ".rs", ".java", ".groovy", ".gradle", ".cpp", ".cc", ".cxx", + ".c", ".h", ".hpp", ".cu", ".cuh", ".metal", ".rb", ".rake", ".swift", + ".kt", ".kts", ".cs", ".scala", ".php", ".lua", ".luau", ".toc", ".zig", + ".ps1", ".psm1", ".psd1", ".ex", ".exs", ".m", ".mm", ".ml", ".mli", ".jl", + ".vue", ".svelte", ".astro", ".dart", ".v", ".sv", ".svh", ".sql", ".r", + ".f", ".F", ".f90", ".F90", ".f95", ".F95", ".f03", ".F03", ".f08", ".F08", + ".pas", ".pp", ".dpr", ".dpk", ".lpr", ".inc", ".dfm", ".lfm", ".lpk", + ".sh", ".bash", ".json", ".tf", ".tfvars", ".hcl", ".dm", ".dme", ".dmi", + ".dmm", ".dmf", ".sln", ".slnx", ".csproj", ".fsproj", ".vbproj", ".xaml", + ".razor", ".cshtml", ".cls", ".trigger", ".lisp", ".cl", ".lsp", ".asd", + ".robot", ".resource", +}) + +#: A manifest row's content hash, as the pin computes it: ``_md5_file`` streams +#: the file and returns a hex digest, or the empty string when the read failed. +#: So a well-formed 32-character digest is the provider's own statement that it +#: read those bytes, and anything else -- blank, short, uppercase, non-string -- +#: is a row that proves nothing about the file it names. +_MANIFEST_HASH = re.compile(r"[0-9a-f]{32}\Z") + +#: The row fields the pinned ``save_manifest`` writes. Read as a shape check +#: only: a mapping missing them is not the document this adapter can classify. +_MANIFEST_ROW_FIELDS = ("mtime", "seen", "ast_hash", "semantic_hash") def _refuse_pre_existing_provider_state(source_root: Path) -> None: @@ -2249,111 +2282,166 @@ def _refuse_pre_existing_provider_state(source_root: Path) -> None: ) -def _provider_state_directory(source_root: Path) -> Path: - """The state directory the provider wrote during this run. +def _provider_output_directory(source_root: Path) -> Path: + """The output root the provider wrote during this run. + + Exactly one name is collected -- the pin's own ``graphify-out`` -- and it is + resolved beneath the materialized copy, never from a path or an environment + variable a caller could supply. Only reachable after + ``_refuse_pre_existing_provider_state``, so what is found here was created + by the run that just finished. - Only reachable after ``_refuse_pre_existing_provider_state``, so whichever - of the two names is present was created by the run that just finished. + A second provider root beside it is refused rather than ignored. This + adapter cannot tell which of two roots a generation should be cut from, and + picking one would publish an artifact whose provenance is a guess; a + ``.graphify/`` that appeared next to ``graphify-out/`` also says the run did + something other than the single contained extraction that was launched. """ - for name in _PROVIDER_STATE_DIRECTORIES: - candidate = source_root / name - if candidate.is_dir() and not candidate.is_symlink(): - return candidate - raise ContextError("local graph provider wrote no index state; no generation was published") + directory = source_root / _PROVIDER_OUTPUT_DIRECTORY + competing = [ + name + for name in _PROVIDER_STATE_DIRECTORIES + if name != _PROVIDER_OUTPUT_DIRECTORY + and ((source_root / name).exists() or (source_root / name).is_symlink()) + ] + if competing: + raise ContextError( + "local graph provider wrote more than one output root; no generation was published" + ) + if directory.is_symlink() or not directory.is_dir(): + raise ContextError("local graph provider wrote no output; no generation was published") + graph = directory / _PROVIDER_GRAPH_NAME + if graph.is_symlink() or not graph.is_file(): + raise ContextError( + "local graph provider left no graph document; no generation was published" + ) + return directory -def _provider_report(state_directory: Path) -> Mapping[str, Any] | None: - """The provider's completion evidence, or ``None`` if it left none. +def _provider_manifest(output_directory: Path) -> Mapping[str, Any] | None: + """The provider's own record of what it processed, or ``None`` if unreadable. - Bounded at the stream, not after the fact: the report is provider output of - unknown size, and reading it whole to slice it afterwards would let it + Bounded at the stream, not after the fact: the manifest is provider output + of unknown size, and reading it whole to slice it afterwards would let it exhaust this process before any budget was consulted. Anything longer than a manifest is rejected outright rather than parsed from a prefix, which would be a different document than the one the provider wrote. """ - for name in _PROVIDER_REPORT_NAMES: - path = state_directory / name - if not path.is_file() or path.is_symlink(): - continue - try: - with path.open("rb") as stream: - raw = stream.read(MAX_MANIFEST_BYTES + 1) - except OSError: - return None - if len(raw) > MAX_MANIFEST_BYTES: - return None - try: - payload = json.loads(raw) - except ValueError: - return None - return payload if isinstance(payload, Mapping) else None - return None - - -def _completion_claim(report: Mapping[str, Any]) -> bool | None: - """``True`` finished, ``False`` denied it, ``None`` said nothing either way.""" - claim: bool | None = None - for name in _COMPLETION_FLAGS: - value = report.get(name) - if value is True: - claim = True - elif value is False: - return False - for field in _COMPLETION_STATUS_FIELDS: - value = report.get(field) - if not isinstance(value, str): - continue - if value.strip().casefold() in _COMPLETION_STATUS_VALUES: - claim = True - else: - # A status the adapter does not recognize is not a completion. - return False - return claim - - -def _indexed_count(report: Mapping[str, Any]) -> int | None: - """How many files the provider says it indexed, if it says at all.""" - for counter in _INDEXED_COUNTERS: - value = report.get(counter) - if isinstance(value, int) and not isinstance(value, bool) and value >= 0: - return value - return None + path = output_directory / _PROVIDER_MANIFEST_NAME + if path.is_symlink() or not path.is_file(): + return None + try: + with path.open("rb") as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + except OSError: + return None + if len(raw) > MAX_MANIFEST_BYTES: + return None + try: + payload = json.loads(raw) + except ValueError: + return None + return payload if isinstance(payload, Mapping) else None -def _read_completeness(report: Mapping[str, Any] | None) -> IndexResult: - """Classify a provider run from its own report, defaulting to partial. +def _eligible_code_inputs(census: TrackedCensus) -> tuple[str, ...]: + """The census paths this pin would dispatch, in census order. - Absent or unreadable evidence is *not* evidence of a complete build, and - neither is a readable report that says nothing. ``complete`` is reached - only by a report shaped the way this adapter understands one: an - affirmative completion claim, a count of what was indexed, and no counter - admitting work left over. An empty object, an unrecognized schema, and a - document that happens to parse all stay ``partial``, which + Case is tried both ways because the pin's set carries both ``.f90`` and + ``.F90``: matching the spelling first and the lower-cased suffix second can + only widen the denominator, which is the direction that refuses rather than + over-claims. + """ + eligible: list[str] = [] + for entry in census.entries: + suffix = PurePosixPath(entry.path).suffix + if suffix in _PROVIDER_CODE_EXTENSIONS or suffix.lower() in _PROVIDER_CODE_EXTENSIONS: + eligible.append(entry.path) + return tuple(eligible) + + +def _read_completeness( + manifest: Mapping[str, Any] | None, census: TrackedCensus | None +) -> IndexResult: + """Classify a provider run against its own manifest, defaulting to partial. + + The pinned ``save_manifest`` writes a flat mapping of repository-relative + POSIX path to ``{mtime, seen, ast_hash, semantic_hash}``. It is not a + completion report and carries no flag or count, so completeness is a + coverage question: did every input this pin would dispatch come back with a + hash proving the provider read its bytes? + + The denominator is the immutable materialized census narrowed to the pin's + own code extensions. Inputs outside that set are deterministically not code + to this pin and are skipped, not missing. Everything else is counted, and a + file is processed only when its row carries a well-formed ``ast_hash``. The + pin blanks that field on exactly the cases an operator needs to hear about: + ``clear_ast`` zeroes both hashes for an extractor error or an anomalous + zero-node extract, so a requeued file is a blank row rather than an absent + one. The clean-room run's 54 requeued entries -- from a repeat that exited + zero in 1.63 s -- are that shape, and they stay partial here. + + Nothing upgrades a run: the exit status, a non-empty graph, and the raw + extraction's ``extracted_sources`` are all statements about what was + *dispatched*, failures included, so none of them is success evidence. A + hash proves processed bytes, not that anything was understood; zero nodes + for a file whose row is stamped is still a complete read of that file, and + an unstamped one is partial however large the graph is. Missing evidence, + an unparseable manifest, a shape this adapter does not recognize, and a row + that cannot be told apart from a failure all stay ``partial``, which ``graph_status`` refuses by default. The provider owns no provenance (the evaluation records this as the first product constraint), so that refusal is the failure an operator can act on; silently calling it complete is the one they cannot. """ - if report is None: - return IndexResult(completeness=PARTIAL, notes=("provider left no readable completion report",)) + if census is None: + return IndexResult( + completeness=PARTIAL, + notes=("build recorded no census to check the provider's manifest against",), + ) + if manifest is None: + return IndexResult( + completeness=PARTIAL, + notes=("provider left no readable manifest of what it processed",), + ) + eligible = _eligible_code_inputs(census) + rows: dict[str, Any] = {} + malformed_rows = 0 + for key, row in manifest.items(): + if not isinstance(key, str): + malformed_rows += 1 + continue + if not isinstance(row, Mapping) or any( + field not in row for field in _MANIFEST_ROW_FIELDS + ): + malformed_rows += 1 + continue + rows[unicodedata.normalize("NFC", key)] = row + missing = 0 + unstamped = 0 + processed = 0 + for path in eligible: + row = rows.get(unicodedata.normalize("NFC", path)) + if row is None: + missing += 1 + continue + digest = row.get("ast_hash") + if isinstance(digest, str) and _MANIFEST_HASH.fullmatch(digest): + processed += 1 + else: + unstamped += 1 notes: list[str] = [] - for counter in _INCOMPLETE_COUNTERS: - value = report.get(counter) - if value is True: - notes.append(f"provider reported {counter}") - elif isinstance(value, int) and not isinstance(value, bool) and value > 0: - notes.append(f"provider reported {value} {counter}") - claim = _completion_claim(report) - if claim is False: - notes.append("provider did not report the extraction as complete") - elif claim is None: - notes.append("provider report carried no completion claim") - indexed = _indexed_count(report) - if indexed is None: - notes.append("provider report did not say how many files it indexed") + if malformed_rows: + notes.append(f"provider manifest carried {malformed_rows} unreadable records") + if missing: + notes.append(f"provider manifest does not account for {missing} code files") + if unstamped: + notes.append(f"provider left {unstamped} code files unprocessed or requeued") + if not eligible: + notes.append("the census carried no code files this provider would index") if notes: - return IndexResult(completeness=PARTIAL, indexed_files=indexed or 0, notes=tuple(notes)) - return IndexResult(completeness=COMPLETE, indexed_files=indexed) + return IndexResult(completeness=PARTIAL, indexed_files=processed, notes=tuple(notes)) + return IndexResult(completeness=COMPLETE, indexed_files=processed) class _BoundedBuffer(io.BytesIO): @@ -2695,9 +2783,9 @@ def run(request: IndexRequest) -> IndexResult: raise ContextError("local graph provider could not be run from its pinned install") from None if returncode != 0: raise ContextError("local graph provider failed; no generation was published") - state_directory = _provider_state_directory(request.source_root) - result = _read_completeness(_provider_report(state_directory)) - _write_private_file(request.output_path, _pack_state(state_directory)) + output_directory = _provider_output_directory(request.source_root) + result = _read_completeness(_provider_manifest(output_directory), request.census) + _write_private_file(request.output_path, _pack_state(output_directory)) return result return run @@ -3558,6 +3646,9 @@ def build_graph( # at, so what the provider is told to use and what it is # allowed to write are one decision rather than two. writable=(home, temporary), + # The very census those bytes were written from, so the + # indexer measures coverage against what it was given. + census=census, ) ) if not isinstance(result, IndexResult) or result.completeness not in (COMPLETE, PARTIAL): diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index 2110a48e..149fcd6b 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -1015,9 +1015,31 @@ def classify(prefix, **keywords): self.assertEqual(calls, [()]) -#: What a provider that finished leaves behind: a completion claim and counts -#: that admit nothing outstanding. -FINISHED_REPORT = {"complete": True, "code_files": 3, "requeued": 0} +#: The census a launch fixture indexes: one Python file, one Markdown file. +#: Only the first is a code input to the pin, so only the first is the +#: denominator completeness is measured against. +CODE_INPUT = "src/app.py" +DOC_INPUT = "docs/guide.md" + + +def census_of(*paths: str) -> lifecycle.TrackedCensus: + """A census naming ``paths``, shaped the way ``read_tracked_census`` does.""" + entries = tuple( + lifecycle.TrackedEntry(path=path, mode="100644", blob="0" * 40, size=1) + for path in paths + ) + return lifecycle.TrackedCensus(entries=entries, skipped=(), digest="0" * 64) + + +def manifest_row(digest: str = "a" * 32) -> dict: + """One ``save_manifest`` row, in the pin's own shape.""" + return {"mtime": 1.0, "seen": 2.0, "ast_hash": digest, "semantic_hash": digest} + + +#: What a provider that processed every code input leaves behind: a row per +#: dispatched file, each carrying the content hash that proves its bytes were +#: read. No completion flag and no count -- the pinned manifest has neither. +FINISHED_MANIFEST = {CODE_INPUT: manifest_row()} class ProviderLaunchTests(TemporaryWorkspace): @@ -1061,6 +1083,7 @@ def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: pin=pin, commit="a" * 40, tree="b" * 40, + census=census_of(CODE_INPUT, DOC_INPUT), ) def run_indexer( @@ -1068,15 +1091,15 @@ def run_indexer( executable: str, *, sandbox=("/sandbox", "--deny"), - report: object = FINISHED_REPORT, - state_directory: str = ".graphify", + report: object = FINISHED_MANIFEST, + state_directory: str = lifecycle._PROVIDER_OUTPUT_DIRECTORY, pin: lifecycle.GraphifyPin = PIN, ) -> tuple[list[str], lifecycle.IndexResult]: """Launch the adapter with the provider's side of the contract faked. - ``extract`` writes its state beside the sources it was run over, so the - stand-in has to leave that state behind for the adapter to collect -- - an exit status alone is not a finished build. + ``extract`` writes ``graphify-out/`` beneath the tree it was run over, + so the stand-in has to leave that output behind for the adapter to + collect -- an exit status alone is not a finished build. """ request = self.request(pin) recorded: list[list[str]] = [] @@ -1090,7 +1113,7 @@ def fake_popen(argv, **kwargs): if state_directory: written = request.source_root / state_directory written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "graph.json").write_text("{}", encoding="utf-8") if report is not None: (written / "manifest.json").write_text(json.dumps(report), encoding="utf-8") return FakeChild() @@ -1262,9 +1285,11 @@ def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: def test_the_collected_artifact_holds_the_state_the_provider_wrote(self) -> None: _, result = self.run_indexer("graphify") self.assertEqual(result.completeness, lifecycle.COMPLETE) - self.assertEqual(result.indexed_files, 3) + # One of the two census files is a code input to this pin; the Markdown + # file is not, so it is skipped rather than counted against the run. + self.assertEqual(result.indexed_files, 1) names = self.archived_names(self.artifacts[-1].read_bytes()) - self.assertIn("graph.bin", names) + self.assertIn("graph.json", names) def archived_names(self, artifact: bytes) -> list[str]: with tarfile.open(fileobj=io.BytesIO(artifact), mode="r") as archive: @@ -1309,30 +1334,145 @@ def test_a_provider_that_wrote_no_state_publishes_nothing(self) -> None: with self.assertRaises(ContextError): self.run_indexer("graphify", state_directory="") + def test_the_output_is_collected_from_the_root_the_pin_actually_writes(self) -> None: + """``graphify-out/``, not a name this adapter would have preferred. + + The pin builds every output path from ``GRAPHIFY_OUT``, whose default is + the literal ``graphify-out``, so a collector that only accepted + ``.graphify``/``.graph`` found nothing after a real run and refused + every generation the adopted path can produce. + """ + _, result = self.run_indexer("graphify") + self.assertEqual(result.completeness, lifecycle.COMPLETE) + with self.assertRaises(ContextError): + self.run_indexer("graphify", state_directory=".graphify") + + def test_a_second_output_root_beside_the_real_one_publishes_nothing(self) -> None: + """Two roots means the provenance of a generation would be a guess.""" + request = self.request() + + def fake_popen(argv, **kwargs): + for name in (lifecycle._PROVIDER_OUTPUT_DIRECTORY, ".graphify"): + written = request.source_root / name + written.mkdir(exist_ok=True) + (written / "graph.json").write_text("{}", encoding="utf-8") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + + def test_a_symlinked_output_root_publishes_nothing(self) -> None: + """A link names a directory outside the copy; the boundary is the copy.""" + request = self.request() + elsewhere = Path(tempfile.mkdtemp(dir=self.root)) + (elsewhere / "graph.json").write_text("{}", encoding="utf-8") + + def fake_popen(argv, **kwargs): + (request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY).symlink_to(elsewhere) + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + + def test_an_output_root_without_a_graph_document_publishes_nothing(self) -> None: + request = self.request() + + def fake_popen(argv, **kwargs): + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY + written.mkdir(exist_ok=True) + (written / "manifest.json").write_text("{}", encoding="utf-8") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + def test_a_successful_run_with_requeued_entries_is_partial(self) -> None: - # The defect the clean-room run recorded: a repeat that exits zero in - # 1.63 s having requeued 54 entries has not built a complete graph. - _, result = self.run_indexer("graphify", report={"complete": True, "files": 429, "requeued": 54}) + """The defect the clean-room run recorded, in the shape the pin writes it. + + A repeat that exits zero in 1.63 s having requeued 54 entries has not + built a complete graph. The pin records a requeue by blanking the row's + hashes -- ``clear_ast`` does exactly that for an extractor error or an + anomalous zero-node extract -- so a blank ``ast_hash`` is the evidence, + not a ``requeued`` counter the pinned manifest has never carried. + """ + requeued = {CODE_INPUT: {"mtime": 1.0, "seen": 2.0, "ast_hash": "", "semantic_hash": ""}} + _, result = self.run_indexer("graphify", report=requeued) self.assertEqual(result.completeness, lifecycle.PARTIAL) - self.assertIn("54 requeued", " ".join(result.notes)) + self.assertIn("1 code files unprocessed or requeued", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) - def test_a_provider_that_denies_completion_is_partial(self) -> None: - _, result = self.run_indexer("graphify", report={"complete": False, "files": 10}) + def test_a_code_input_the_manifest_never_names_is_partial(self) -> None: + """Silence about a file is not a claim that it was indexed.""" + _, result = self.run_indexer("graphify", report={}) self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) - def test_a_run_that_left_no_report_is_partial_rather_than_complete(self) -> None: + def test_a_non_code_input_is_skipped_rather_than_counted_against_the_run(self) -> None: + """The denominator is the pin's own code set, not every tracked file. + + ``docs/guide.md`` is deterministically not code to this pin, so a run + that never touches it is still complete. Holding a correct run to every + tracked documentation file would make it permanently partial. + """ + _, result = self.run_indexer("graphify") + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertNotIn(DOC_INPUT, json.dumps(FINISHED_MANIFEST)) + + def test_a_malformed_manifest_record_is_partial_rather_than_complete(self) -> None: + """A row this adapter cannot read is indistinguishable from a failure.""" + malformed = ( + {CODE_INPUT: "not a row"}, + {CODE_INPUT: {"ast_hash": "a" * 32}}, + {CODE_INPUT: manifest_row("A" * 32)}, + {CODE_INPUT: manifest_row("short")}, + {CODE_INPUT: dict(manifest_row(), ast_hash=None)}, + ) + for report in malformed: + with self.subTest(report=report): + _, result = self.run_indexer("graphify", report=report) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertTrue(result.notes) + + def test_a_manifest_listing_every_code_input_with_a_hash_is_complete(self) -> None: + """Hashes prove processed bytes, which is what completeness claims. + + Nothing else upgrades a run: the exit status is zero in every case here, + and the graph document is ``{}`` -- an empty graph with every input + stamped is a complete read, and a full graph with one input unstamped is + not. + """ + _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row("b" * 32)}) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + + def test_a_request_without_a_census_cannot_be_complete(self) -> None: + """There is no denominator, so there is no coverage claim to make.""" + self.assertEqual( + lifecycle._read_completeness(FINISHED_MANIFEST, None).completeness, lifecycle.PARTIAL + ) + + def test_a_run_that_left_no_manifest_is_partial_rather_than_complete(self) -> None: # Exit status zero is not completion evidence. Absent evidence resolves # to the state ``graph_status`` refuses, not the one it accepts. _, result = self.run_indexer("graphify", report=None) self.assertEqual(result.completeness, lifecycle.PARTIAL) - def test_an_unparseable_report_is_partial_rather_than_complete(self) -> None: + def test_an_unparseable_manifest_is_partial_rather_than_complete(self) -> None: request = self.request() def fake_popen(argv, **kwargs): - written = request.source_root / ".graph" + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "graph.json").write_text("{}", encoding="utf-8") (written / "manifest.json").write_bytes(b"{not json") return FakeChild() @@ -1342,40 +1482,7 @@ def fake_popen(argv, **kwargs): result = indexer(request) self.assertEqual(result.completeness, lifecycle.PARTIAL) - def test_a_report_without_affirmative_completion_evidence_is_partial(self) -> None: - """A document that parses is not a document that claims completion. - - ``{}`` and a report in some schema this adapter does not understand - both say nothing about whether the extraction finished, and nothing is - not a claim. Treating them as complete would hand ``graph_status`` a - usable generation built from an unknown run. - """ - silent = ( - {}, - {"schema": "unexpected"}, - {"code_files": 3}, - {"complete": True}, - {"status": "running", "code_files": 3}, - {"status": "partial", "complete": True, "code_files": 3}, - ) - for report in silent: - with self.subTest(report=report): - _, result = self.run_indexer("graphify", report=report) - self.assertEqual(result.completeness, lifecycle.PARTIAL) - self.assertTrue(result.notes) - - def test_a_report_that_claims_completion_and_counts_its_work_is_complete(self) -> None: - claimed = ( - {"complete": True, "code_files": 3}, - {"status": "success", "indexed_files": 3}, - {"completed": True, "entries": 0, "requeued": 0}, - ) - for report in claimed: - with self.subTest(report=report): - _, result = self.run_indexer("graphify", report=report) - self.assertEqual(result.completeness, lifecycle.COMPLETE) - - def test_an_oversized_report_is_refused_without_being_read_whole(self) -> None: + def test_an_oversized_manifest_is_refused_without_being_read_whole(self) -> None: """Provider output is unbounded input; the read is bounded at the stream. Slicing after ``read_bytes()`` would have allocated the whole document @@ -1383,12 +1490,12 @@ def test_an_oversized_report_is_refused_without_being_read_whole(self) -> None: nothing in the collection path may read a provider file whole. """ request = self.request() - oversized = b'{"complete": true, "code_files": 3, "pad": "' + b"x" * lifecycle.MAX_MANIFEST_BYTES + b'"}' + oversized = b'{"' + CODE_INPUT.encode() + b'": "' + b"x" * lifecycle.MAX_MANIFEST_BYTES + b'"}' def fake_popen(argv, **kwargs): - written = request.source_root / ".graphify" + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "graph.json").write_text("{}", encoding="utf-8") (written / "manifest.json").write_bytes(oversized) return FakeChild() @@ -2164,6 +2271,17 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: really links against end up inside the boundary. Every absolute dependency that extension names must be either under the read-only runtime every child gets or in the derived set. + + Not every host runtime can be admitted, and this test must not assume + it. An interpreter installed under a shared package-manager prefix whose + ancestry is group-writable is a runtime another account may rewrite, and + the trust rule refuses it on purpose: the derivation raises rather than + exposing code somebody else chooses the contents of. So a refusal is + checked here rather than swallowed -- this host is asked, independently + of the derivation, whether one of ``_ssl``'s own dependencies really is + untrusted, and the refusal is only accepted when it is. A refusal over a + runtime this host does trust would be the regression this test exists to + catch, and any other exception is not caught at all. """ import _ssl # noqa: PLC0415 - the point is this host's real extension @@ -2171,18 +2289,46 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: if not extension.is_file(): # pragma: no cover - a statically linked build self.skipTest("this interpreter's _ssl is not a separate extension module") prefix = Path(sys.base_prefix) - derived = lifecycle._linked_runtime_libraries( - [prefix], covered=[prefix], repository=self.repository - ) system = [Path(os.path.realpath(path)) for path in lifecycle._SYSTEM_READ_PATHS] covered = [*system, Path(os.path.realpath(prefix))] - for name in lifecycle._macho_dylib_names(extension): - if not name.startswith("/"): - continue - resolved = Path(os.path.realpath(name)) - if any(lifecycle._under(resolved, root) for root in covered): - continue - self.assertIn(str(resolved), derived, f"{name} is outside the provider's boundary") + # The extension's own out-of-runtime dependencies, resolved the way the + # derivation resolves them. Computed before the derivation runs so the + # expectation does not come from the code under test. + external = [ + Path(os.path.realpath(name)) + for name in lifecycle._macho_dylib_names(extension) + if name.startswith("/") + ] + external = [ + resolved + for resolved in external + if not any(lifecycle._under(resolved, root) for root in covered) + ] + untrusted = [str(path) for path in external if not lifecycle._trusted_library(path)] + try: + derived = lifecycle._linked_runtime_libraries( + [prefix], covered=[prefix], repository=self.repository + ) + except lifecycle.ContextError as refusal: + # The derivation walks the whole runtime, so the library it refused + # need not be one of ``_ssl``'s: a shared package-manager prefix is + # refused through its own ancestry, before any single dependency. + # Either is a correct refusal; neither being true is not. + self.assertTrue( + untrusted or not lifecycle._trusted_library(Path(os.path.realpath(prefix))), + "the boundary refused this host's runtime, but the runtime and " + f"every library _ssl links to are trusted: {refusal}", + ) + return + self.assertEqual( + untrusted, + [], + "the boundary admitted a runtime whose libraries are writable by another account", + ) + for resolved in external: + self.assertIn( + str(resolved), derived, f"{resolved} is outside the provider's boundary" + ) class ProviderIdentityTests(TemporaryWorkspace): @@ -3385,10 +3531,26 @@ def indexer_script(self, *, complete: bool = True, **installed) -> Path: '[ -n "$2" ] || exit 65\n' 'case "$2" in -*) exit 65 ;; esac\n' '[ -d "$2" ] || exit 65\n' - "mkdir -p .graphify\n" - "printf graph-bytes > .graphify/graph.bin\n" - 'printf \'{"complete": %s, "code_files": 1, "requeued": 0}\' ' - f"'{'true' if complete else 'false'}' > .graphify/manifest.json\n" + # The real output root, with the two documents the pinned + # ``--no-cluster`` branch writes there. + "mkdir -p graphify-out\n" + 'printf \'{"nodes": [], "edges": []}\' > graphify-out/graph.json\n' + # A complete run stamps every input it was shown; a partial one + # leaves the manifest empty, which is what an unprocessed code + # file looks like to the collector. + + ( + ( + "{ printf '{'; sep=''; " + "find . -path ./graphify-out -prune -o -type f -print | " + "sed 's|^\\./||' | while read -r f; do " + 'printf \'%s"%s":{"mtime":1,"seen":2,' + '"ast_hash":"0123456789abcdef0123456789abcdef",' + "\"semantic_hash\":\"0123456789abcdef0123456789abcdef\"}' \"$sep\" \"$f\"; " + "sep=','; done; printf '}'; } > graphify-out/manifest.json\n" + ) + if complete + else "printf '{}' > graphify-out/manifest.json\n" + ) ), **installed, ) From d48444ffe59c9ffe02abf4262d62ca6147086a00 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 08:37:19 -0700 Subject: [PATCH 12/33] Graph lifecycle tests: finish the output-root rename in the exposure test The exposure fixture still wrote .graphify/graph.bin and the old report shape, so it exercised a collector path that no longer exists. Co-Authored-By: Claude Opus 5 (1M context) --- tests/test_context_graph_lifecycle.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index 149fcd6b..d74df92a 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -1156,10 +1156,10 @@ def record_prefix(**keywords: object) -> tuple[str, ...]: return ("/sandbox",) def fake_popen(argv, **kwargs): - written = request.source_root / ".graphify" + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY written.mkdir(exist_ok=True) - (written / "graph.bin").write_bytes(b"graph-bytes") - (written / "manifest.json").write_text(json.dumps(FINISHED_REPORT), encoding="utf-8") + (written / "graph.json").write_text("{}", encoding="utf-8") + (written / "manifest.json").write_text(json.dumps(FINISHED_MANIFEST), encoding="utf-8") return FakeChild() with stand_in_containment(("/sandbox",)): From 71896ed75a8afbefc9633b3a396d809f8e795720 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 08:51:28 -0700 Subject: [PATCH 13/33] Graph queries: bare callable seeds, and hashes checked against the bytes Two acceptance repairs on the same branch. 1. A bare symbol now resolves to the provider's canonical callable label. The pinned 0.9.58 extractor labels a callable with its argument list -- a fresh contained extraction of the public fixture names the function parse_graph_citation as `parse_graph_citation()` and packet as `packet()` -- so an exact-label-only reader answered "unresolved" for every natural spelling of a function name. Resolution is now three ordered tiers, later consulted only when earlier ones are empty: the literal label, then the label with only its syntactic trailing argument list removed, then the path. Tier two is an equality test against a stripped label, not a prefix, substring, or distance match: the suffix must begin at the first `(`, be balanced, and close on the last character. `parse_graph` does not reach `parse_graph_citation()`; an attribute `packet` and a function `packet()` stay distinct and the exact label wins; overload-like labels both resolve, which is real ambiguity and is reported rather than settled by picking one. Path handling, the seed bound, and its truncation are unchanged. 2. A manifest row is now checked against the bytes the provider was given. `_read_completeness` treated any 32-hex ast_hash as proof of work. The pin's `_md5_file` returns the MD5 hex digest of a file's contents, so the adapter takes that digest of every eligible code input from the materialized copy before the launch -- the only moment that tree is still exactly what the provider was handed -- and requires the row to match. A well-formed digest alone says a hash-shaped string is present; the comparison is what says it is a hash of this input. A disagreeing hash, an input the copy cannot be re-read for, and a build that recorded no digests all stay partial. Nothing upgrades a run. Docs for both, plus the stale completeness and output-root prose the previous rounds left behind. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-lifecycle.md | 72 +++++++++---- docs/context-graph-queries.md | 19 ++++ src/code_mower/context_graph_lifecycle.py | 119 ++++++++++++++++++++-- src/code_mower/context_graph_query.py | 69 +++++++++++++ tests/test_context_graph_lifecycle.py | 106 ++++++++++++++++++- tests/test_context_graph_query.py | 99 ++++++++++++++++++ 6 files changed, 452 insertions(+), 32 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index eb71342f..3e751f60 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -44,9 +44,10 @@ provenance at all. If Code Mower does not bind the revision, nothing does. repository it was never authorized to read. Committed private state is skipped for a third reason, at any depth and case-folded: the roots are `context_graph`'s excluded roots themselves — `.git`, `.graph`, `.graphify`, - `.code-mower` — bound rather than copied, so the set that refuses a citation - into private state is the same set that keeps those bytes away from the - indexer. A tracked `.graphify/` or `.graph/` is somebody's old index, and + `graphify-out`, `.code-mower` — bound rather than copied, so the set that + refuses a citation into private state is the same set that keeps those bytes + away from the indexer. A tracked `.graphify/`, `.graph/` or `graphify-out/` + is somebody's old index, and materializing it would let the provider resume from a cache built over content this build never saw, and let the adapter collect tracked repository bytes as if the provider had just produced them. A tracked `.code-mower/` @@ -361,8 +362,12 @@ valued form such as `--code-only=false` — is refused rather than quietly overridden by argument order. So the adapter collects an artifact afterwards rather than naming one up front. -The state directory the provider wrote (`.graphify` or `.graph`, both already on -the excluded-roots list) is packed into a single reproducible archive: names +The output root the provider wrote — exactly `graphify-out`, the literal default +of the pin's own `GRAPHIFY_OUT`, resolved beneath the materialized copy and +never from a caller path or that environment override, and required to be a real +non-symlinked directory holding `graph.json` as a regular file, with a second +provider root beside it refused rather than guessed between — is packed into a +single reproducible archive: names sorted, timestamps and ownership fixed, modes normalized, symlinks dropped. Two builds of one commit have to produce identical bytes, because the manifest binds a digest of them. That state lands inside the throwaway materialized copy, never @@ -382,26 +387,53 @@ build from this module there is none; the refusal is the second check, because everything after the run treats whatever is in that directory as output this run produced. -**Completeness is read from the provider's report, never from its exit status, -and only an affirmative claim counts.** `complete` requires a report shaped the -way the adapter understands one: a claim that the run finished (`complete`, -`completed`, `finished`, or a recognized `status`), a count of what was -indexed, and no counter admitting requeued, pending, or failed work. Everything -else is `partial` — a report that denies completion, one in an unrecognized -schema, an empty object, an unreadable one, one larger than a manifest, and no -report at all. Absent evidence is not evidence of a complete build, and -`partial` is the state `graph_status` refuses by default, so the failure is one -an operator can see and act on. This is the direct consequence of the requeue -defect the evaluation recorded — a repeat that exits zero in 1.63 seconds -having requeued 54 entries has not built a complete graph. - -The report is provider output of unknown size, so it is read to one byte past +**Completeness is coverage, read from the provider's own manifest and never +from its exit status.** The pinned `save_manifest` writes a flat map of +repository-relative POSIX path to `{mtime, seen, ast_hash, semantic_hash}`. It +carries no completion flag and no indexed count, so there is no affirmative +claim to accept: the question is whether every input this pin would dispatch +came back with a hash proving the provider read its bytes. + +The denominator is the immutable materialized census narrowed to the pin's own +`detect.CODE_EXTENSIONS`. Inputs outside that set are deterministically not code +to this pin and are skipped rather than counted missing — holding a correct run +to every tracked Markdown file would make it permanently `partial`. + +A file counts as processed only when its row carries a well-formed 32-character +`ast_hash` **and that hash is the digest of the bytes this build actually handed +the provider**. The pin's `_md5_file` streams a file and returns the MD5 hex +digest of its contents, so the adapter takes the same digest of every eligible +input from the materialized copy *before* the launch — the only moment that tree +is still exactly what the provider was given — and compares. A well-formed +digest on its own only says a hash-shaped string is present; the comparison is +what says it is a hash of this input, and without it a manifest carried over +from another tree, another revision, or a resumed cache would read as proof of +work on bytes the provider never saw. + +Everything else is `partial`: a row the manifest never wrote, a blank or +malformed hash, a hash that disagrees with those bytes, an input the copy could +not be re-read for, a record this adapter cannot classify, an unreadable or +oversized manifest, no manifest at all, and a build with no census. The pin +blanks hashes through `clear_ast` on an extractor error or an anomalous +zero-node extract, so a requeued file is a blank row rather than an absent one +— the direct consequence of the requeue defect the evaluation recorded, where a +repeat that exits zero in 1.63 seconds having requeued 54 entries has not built +a complete graph. + +Nothing upgrades a run. The exit status, a non-empty graph, and the raw +extraction's `extracted_sources` all describe what was *dispatched*, failures +included. A hash proves processed bytes, not that anything was understood: zero +nodes for a stamped file is still a complete read of it, and an unstamped file +is `partial` however large the graph is. `partial` is the state `graph_status` +refuses by default, so the failure is one an operator can see and act on. + +The manifest is provider output of unknown size, so it is read to one byte past the manifest bound and refused if it is longer, rather than loaded whole and measured afterwards. A bound checked on bytes already in memory bounds nothing. The provider's own stdout and stderr are the other unbounded output, and they are discarded at the kernel: `stdin`, `stdout` and `stderr` are all -`DEVNULL`. Nothing reads them — completeness comes from the report, not from +`DEVNULL`. Nothing reads them — completeness comes from the manifest, not from what the run printed — so buffering them would only accumulate whatever a talkative indexer chose to log, for up to the timeout, under neither the tracked-content budget nor the artifact one. Inheriting them is not the diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index da9fce13..96fc802c 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -168,6 +168,25 @@ breadth-first over adjacency sorted by `(kind, target, source)`, so one generation and one question produce one answer, every time, and a budget cut removes the furthest relationships rather than arbitrary ones. +A target resolves in three ordered tiers, and a later tier is consulted only +when every earlier one is empty: + +1. the provider's label, exactly as written; +2. the provider's canonical callable label with only its syntactic trailing + argument list removed; +3. the path, exactly as written. + +Tier 2 exists because the pinned extractor labels a callable with its argument +list: a real 0.9.58 graph of this repository names the function +`parse_graph_citation` as `parse_graph_citation()`. Without it, the natural bare +spelling of a function resolves to nothing and every question about it answers +"unresolved". It is an equality test against a label with one suffix stripped — +not a prefix, substring, or edit-distance match. `parse_graph` does not reach +`parse_graph_citation()`; an attribute `packet` and a function `packet()` stay +different definitions, and the exact label wins; and two overloads that differ +only in their argument lists both resolve, which is real ambiguity and is +reported as `unresolved_entities` rather than settled by picking one. + Reaching the node budget sets `truncated` and raises `provider_has_more`. It never silently shortens the answer. A target name carried by more definitions than the seed bound allows does the same: seeds the bound drops take their whole diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index a5a0c929..4557a1ef 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -2262,6 +2262,73 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: #: only: a mapping missing them is not the document this adapter can classify. _MANIFEST_ROW_FIELDS = ("mtime", "seen", "ast_hash", "semantic_hash") +#: How much of a materialized input is hashed at a time while deriving what the +#: provider's row for it must say. Bounded by the census, which is already +#: bounded by ``MAX_TRACKED_BYTES``, so this only bounds resident memory. +_MANIFEST_DIGEST_CHUNK_BYTES = 1024 * 1024 + + +def _materialized_digest(path: Path) -> str: + """The pin's own content digest of one materialized input, or ``""``. + + ``_md5_file`` in the pinned 0.9.58 wheel streams the file and returns the + MD5 hex digest of its bytes -- the same digest the fresh contained + extraction's manifest carried for both Python inputs of the public fixture, + matching their exact bytes. So this is not a re-implementation of the + provider's AST work: it is the one thing the provider's row is a statement + *about*, computed here so the statement can be checked rather than believed. + + ``""`` for anything that cannot be read as a regular file, which is a + refusal rather than a pass: a row this build cannot check is a row it cannot + count. + """ + digest = hashlib.md5(usedforsecurity=False) + try: + if path.is_symlink() or not path.is_file(): + return "" + with path.open("rb") as stream: + while True: + chunk = stream.read(_MANIFEST_DIGEST_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + except OSError: + return "" + return digest.hexdigest() + + +def _materialized_digests( + source_root: Path, census: TrackedCensus | None +) -> dict[str, str] | None: + """What every eligible code input's manifest row must say, before the run. + + Taken from the materialized copy *before* the provider is launched, which is + the only moment those bytes are still exactly what this build gave it. After + the run the same tree also holds provider output, and a digest read then + would be checking the provider's manifest against whatever the provider left + behind. + + ``None`` when there is no census, because there is then nothing to enumerate + -- that case is already the partial one. An input the copy cannot be read + for is simply absent from the map, and ``_read_completeness`` keeps the run + partial for it rather than accepting the row unchecked. + """ + if census is None: + return None + digests: dict[str, str] = {} + for path in _eligible_code_inputs(census): + parts = PurePosixPath(path).parts + # ``read_tracked_census`` reads Git's own index paths, which are + # relative and carry no traversal segment. Held to that here anyway, + # because this is the one place a census path is turned back into a + # host path, and a join is not the place to discover otherwise. + if not parts or any(part in ("", ".", "..", "/") for part in parts): + continue + digest = _materialized_digest(source_root.joinpath(*parts)) + if digest: + digests[unicodedata.normalize("NFC", path)] = digest + return digests + def _refuse_pre_existing_provider_state(source_root: Path) -> None: """Refuse to extract on top of index state this build did not produce. @@ -2361,7 +2428,9 @@ def _eligible_code_inputs(census: TrackedCensus) -> tuple[str, ...]: def _read_completeness( - manifest: Mapping[str, Any] | None, census: TrackedCensus | None + manifest: Mapping[str, Any] | None, + census: TrackedCensus | None, + digests: Mapping[str, str] | None = None, ) -> IndexResult: """Classify a provider run against its own manifest, defaulting to partial. @@ -2374,7 +2443,15 @@ def _read_completeness( The denominator is the immutable materialized census narrowed to the pin's own code extensions. Inputs outside that set are deterministically not code to this pin and are skipped, not missing. Everything else is counted, and a - file is processed only when its row carries a well-formed ``ast_hash``. The + file is processed only when its row carries a well-formed ``ast_hash`` *and + that hash is the digest of the bytes this build actually handed the + provider*. A well-formed digest alone says a hash-shaped string is present; + only the comparison says it is a hash of this input. Without it a row + carried over from another tree, another revision, or a resumed cache reads + as proof of work on bytes the provider was never shown -- and the digests + come from ``_materialized_digests``, taken before the launch, so they cannot + have been influenced by what the run wrote. A row whose digest disagrees, and + an input the copy could not be re-read for, both stay partial. The pin blanks that field on exactly the cases an operator needs to hear about: ``clear_ast`` zeroes both hashes for an extractor error or an anomalous zero-node extract, so a requeued file is a blank row rather than an absent @@ -2404,6 +2481,11 @@ def _read_completeness( completeness=PARTIAL, notes=("provider left no readable manifest of what it processed",), ) + if digests is None: + return IndexResult( + completeness=PARTIAL, + notes=("build recorded no input digests to check the provider's manifest against",), + ) eligible = _eligible_code_inputs(census) rows: dict[str, Any] = {} malformed_rows = 0 @@ -2419,17 +2501,29 @@ def _read_completeness( rows[unicodedata.normalize("NFC", key)] = row missing = 0 unstamped = 0 + unreadable = 0 + mismatched = 0 processed = 0 for path in eligible: - row = rows.get(unicodedata.normalize("NFC", path)) + key = unicodedata.normalize("NFC", path) + row = rows.get(key) if row is None: missing += 1 continue digest = row.get("ast_hash") - if isinstance(digest, str) and _MANIFEST_HASH.fullmatch(digest): - processed += 1 - else: + if not isinstance(digest, str) or not _MANIFEST_HASH.fullmatch(digest): unstamped += 1 + continue + expected = digests.get(key) + if expected is None: + # The row is well formed and this build cannot say what it should + # have contained. Counting it would be believing the row on its own + # word, which is the whole thing the comparison exists to stop. + unreadable += 1 + elif digest != expected: + mismatched += 1 + else: + processed += 1 notes: list[str] = [] if malformed_rows: notes.append(f"provider manifest carried {malformed_rows} unreadable records") @@ -2437,6 +2531,10 @@ def _read_completeness( notes.append(f"provider manifest does not account for {missing} code files") if unstamped: notes.append(f"provider left {unstamped} code files unprocessed or requeued") + if unreadable: + notes.append(f"build could not re-read {unreadable} code files to check their hashes") + if mismatched: + notes.append(f"provider hashed {mismatched} code files that are not the bytes it was given") if not eligible: notes.append("the census carried no code files this provider would index") if notes: @@ -2752,6 +2850,11 @@ def run(request: IndexRequest) -> IndexResult: "against; no generation was published" ) _refuse_pre_existing_provider_state(request.source_root) + # Before the launch, and only here. These are the bytes this build hands + # the provider; once the child has run, the same tree also holds the + # provider's own output, and a digest taken then would be checking the + # provider's manifest against the provider's own leavings. + digests = _materialized_digests(request.source_root, request.census) # Built per run, because the boundary is a function of what this build # exposes: the materialized copy and the build's own scratch areas are # writable, the pinned provider's install is readable, and nothing else @@ -2784,7 +2887,9 @@ def run(request: IndexRequest) -> IndexResult: if returncode != 0: raise ContextError("local graph provider failed; no generation was published") output_directory = _provider_output_directory(request.source_root) - result = _read_completeness(_provider_manifest(output_directory), request.census) + result = _read_completeness( + _provider_manifest(output_directory), request.census, digests + ) _write_private_file(request.output_path, _pack_state(output_directory)) return result diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index d3fbd94c..b2fc5319 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -251,6 +251,53 @@ #: large generated file costs the lines up to the claim. _BLOB_CHUNK_BYTES = 256 * 1024 +#: The syntactic argument list the pinned extractor appends to a callable's +#: label. A real 0.9.58 graph of this repository labels the function +#: ``parse_graph_citation`` as ``parse_graph_citation()`` and the function +#: ``packet`` as ``packet()``, so the label a human would type and the label the +#: provider wrote differ by exactly this suffix. +_CALLABLE_OPEN = "(" +_CALLABLE_CLOSE = ")" + + +def _callable_base(label: str) -> str: + """A callable label's name with only its trailing argument list removed. + + ``""`` for anything that is not a callable label, and that emptiness is + load-bearing: ``seed_matches`` compares this against a target that ``_text`` + has already rejected as empty, so a non-callable label can never match by + both sides being blank. + + Deliberately syntactic and deliberately narrow. The suffix must begin at the + label's *first* ``(``, must be a balanced parenthesized group, and that group + must close on the label's last character; the name before it must be + non-empty and carry no ``)`` of its own. So ``render(int)``, + ``render(Callable[(int)])`` and ``render()`` all reduce to ``render`` while + ``render(int))``, ``render()x``, ``render(int`` and ``()`` reduce to nothing. + It is not a parser: the pinned extractor writes the label, this reads the one + suffix it writes, and everything else stays an exact comparison. + """ + if not label.endswith(_CALLABLE_CLOSE): + return "" + opened = label.find(_CALLABLE_OPEN) + if opened <= 0: + return "" + base = label[:opened] + if _CALLABLE_CLOSE in base: + return "" + depth = 0 + for index in range(opened, len(label)): + character = label[index] + if character == _CALLABLE_OPEN: + depth += 1 + elif character == _CALLABLE_CLOSE: + depth -= 1 + if depth == 0: + # The first group closes here. Anything after it means the tail + # is not one argument list, so the label is left alone. + return base if index == len(label) - 1 else "" + return "" + @dataclass(frozen=True) class GraphNode: @@ -322,6 +369,24 @@ def seed_matches(self, target: str) -> tuple[tuple[GraphNode, ...], bool]: all is read as a path. Ordered by id so two runs against one generation seed identically. + Resolution is three ordered tiers, and a later tier is consulted only + when every earlier one is empty, so a literal label always wins over + the same string read as a bare callable and both win over a path: + + 1. The provider's label, exactly as written. + 2. The provider's *canonical callable label* with only its syntactic + trailing argument list removed (``_callable_base``). A real pinned + graph labels a function ``parse_graph_citation()``, so without this + tier the natural bare spelling of a function name resolves to + nothing and every query about it answers "unresolved". + 3. The path, exactly as written. + + Tier 2 is an equality test against a label with one suffix stripped -- + not a prefix, substring, or edit-distance match. ``parse_graph`` does + not reach ``parse_graph_citation()``, and two overloads that differ + only in their argument lists both match, which is real ambiguity and is + reported as such rather than resolved by picking one. + The overflow flag is not cosmetic. A name carried by more than ``MAX_SEEDS`` definitions has definitions this traversal will never start from, and every relationship reachable only from those is absent @@ -331,6 +396,10 @@ def seed_matches(self, target: str) -> tuple[tuple[GraphNode, ...], bool]: """ name = _text(target, maximum=512) matches = [node for node in self.nodes.values() if node.name == name] + if not matches: + matches = [ + node for node in self.nodes.values() if _callable_base(node.name) == name + ] if not matches: matches = [node for node in self.nodes.values() if node.path == name] ordered = tuple(sorted(matches, key=lambda node: node.id)) diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index d74df92a..75d4c213 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -1021,17 +1021,47 @@ def classify(prefix, **keywords): CODE_INPUT = "src/app.py" DOC_INPUT = "docs/guide.md" +#: The bytes a launch fixture actually materializes for each of those, because +#: a manifest row is now checked against them rather than merely shaped like a +#: hash. A census that named files the copy does not hold would be a build that +#: cannot check any row, which is its own (partial) case and has its own test. +FIXTURE_INPUTS = { + CODE_INPUT: b"def main():\n return 0\n", + DOC_INPUT: b"# guide\n", +} + +#: What the pin's ``_md5_file`` would return for the code input above. The +#: adapter derives the same value from the copy before the launch, so this is +#: the one digest a finished manifest can carry for ``src/app.py``. +CODE_INPUT_DIGEST = hashlib.md5(FIXTURE_INPUTS[CODE_INPUT], usedforsecurity=False).hexdigest() + def census_of(*paths: str) -> lifecycle.TrackedCensus: """A census naming ``paths``, shaped the way ``read_tracked_census`` does.""" entries = tuple( - lifecycle.TrackedEntry(path=path, mode="100644", blob="0" * 40, size=1) + lifecycle.TrackedEntry( + path=path, mode="100644", blob="0" * 40, size=len(FIXTURE_INPUTS.get(path, b"x")) + ) for path in paths ) return lifecycle.TrackedCensus(entries=entries, skipped=(), digest="0" * 64) -def manifest_row(digest: str = "a" * 32) -> dict: +def materialize_fixture_inputs(source_root: Path, census: lifecycle.TrackedCensus) -> None: + """Write the census's own bytes into the copy, as a real build would. + + ``materialize_tracked_files`` puts the commit's blobs here before the + provider is launched; the launch fixtures used to leave the copy empty, + which no longer describes a build whose completeness check reads those + bytes. + """ + for entry in census.entries: + destination = source_root.joinpath(*entry.path.split("/")) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(FIXTURE_INPUTS.get(entry.path, b"x")) + + +def manifest_row(digest: str = CODE_INPUT_DIGEST) -> dict: """One ``save_manifest`` row, in the pin's own shape.""" return {"mtime": 1.0, "seen": 2.0, "ast_hash": digest, "semantic_hash": digest} @@ -1076,6 +1106,11 @@ def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: # than exercise anything, so each request names its own. artifact = Path(tempfile.mkdtemp(dir=self.root)) / "graph.bin" self.artifacts.append(artifact) + census = census_of(CODE_INPUT, DOC_INPUT) + # A real build materializes the census into the copy before launching. + # The completeness check now reads those bytes, so the fixture has to + # actually hold them rather than name them. + materialize_fixture_inputs(source, census) return lifecycle.IndexRequest( source_root=source, output_path=artifact, @@ -1083,7 +1118,7 @@ def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: pin=pin, commit="a" * 40, tree="b" * 40, - census=census_of(CODE_INPUT, DOC_INPUT), + census=census, ) def run_indexer( @@ -1448,18 +1483,79 @@ def test_a_manifest_listing_every_code_input_with_a_hash_is_complete(self) -> No Nothing else upgrades a run: the exit status is zero in every case here, and the graph document is ``{}`` -- an empty graph with every input stamped is a complete read, and a full graph with one input unstamped is - not. + not. The hash has to be *this* input's, which is the next test. """ - _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row("b" * 32)}) + _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row()}) self.assertEqual(result.completeness, lifecycle.COMPLETE) self.assertEqual(result.indexed_files, 1) + def test_a_hash_that_is_not_this_input_s_bytes_is_partial(self) -> None: + """A hash-shaped string is not a hash of what the provider was given. + + ``b`` repeated is well formed by every rule the old check applied, and it + is not the digest of ``src/app.py``. A row like it is what a manifest + carried over from another tree or resumed from a cache looks like, so it + must not count as work done on these bytes. + """ + _, result = self.run_indexer("graphify", report={CODE_INPUT: manifest_row("b" * 32)}) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("not the bytes it was given", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) + + def test_the_expected_digest_is_taken_before_the_provider_runs(self) -> None: + """The comparison is against what this build handed over, not leavings. + + The stand-in rewrites ``src/app.py`` while it "runs" and stamps the + manifest with the digest of what it wrote. Read after the fact, that + agrees with itself; read before the launch, as it is, it does not. + """ + request = self.request() + replacement = b"def main():\n return 1\n" + stamped = hashlib.md5(replacement, usedforsecurity=False).hexdigest() + + def fake_popen(argv, **kwargs): + request.source_root.joinpath(*CODE_INPUT.split("/")).write_bytes(replacement) + written = request.source_root / lifecycle._PROVIDER_OUTPUT_DIRECTORY + written.mkdir(exist_ok=True) + (written / "graph.json").write_text("{}", encoding="utf-8") + (written / "manifest.json").write_text( + json.dumps({CODE_INPUT: manifest_row(stamped)}), encoding="utf-8" + ) + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + result = indexer(request) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("not the bytes it was given", " ".join(result.notes)) + + def test_a_code_input_the_copy_does_not_hold_cannot_be_counted(self) -> None: + """An unreadable input is a row this build cannot check, so it is partial. + + Not an error: the census is the denominator and the copy is the + evidence, and a build that lost one of the two states that rather than + believing the provider's row on its own word. + """ + result = lifecycle._read_completeness( + {CODE_INPUT: manifest_row()}, census_of(CODE_INPUT), {} + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not re-read 1 code files", " ".join(result.notes)) + self.assertEqual(result.indexed_files, 0) + def test_a_request_without_a_census_cannot_be_complete(self) -> None: """There is no denominator, so there is no coverage claim to make.""" self.assertEqual( lifecycle._read_completeness(FINISHED_MANIFEST, None).completeness, lifecycle.PARTIAL ) + def test_a_build_that_recorded_no_input_digests_cannot_be_complete(self) -> None: + """Without the immutable bytes there is nothing to check a row against.""" + result = lifecycle._read_completeness(FINISHED_MANIFEST, census_of(CODE_INPUT), None) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("no input digests", " ".join(result.notes)) + def test_a_run_that_left_no_manifest_is_partial_rather_than_complete(self) -> None: # Exit status zero is not completion evidence. Absent evidence resolves # to the state ``graph_status`` refuses, not the one it accepts. diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 991bc6a4..d3fd0ed4 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -676,6 +676,105 @@ def test_an_unsupported_question_is_refused(self) -> None: self.query(question="everything") +class CallableLabelSeedTests(unittest.TestCase): + """Bare names against the labels the pinned extractor actually writes. + + A fresh contained extraction of the public fixture at 0.9.58 labels the + function ``parse_graph_citation`` as ``parse_graph_citation()`` and the + function ``packet`` as ``packet()``. An exact-label-only reader answers + "unresolved" for every natural spelling of a function name, so a bare symbol + resolves to the canonical callable label with only its syntactic trailing + argument list removed -- and to nothing else. These tests are as much about + what that must *not* reach. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_a_bare_name_resolves_to_the_canonical_callable_label(self) -> None: + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual([item.id for item in graph.seeds("parse_graph_citation")], ["n-citation"]) + + def test_the_literal_label_still_resolves_exactly(self) -> None: + """Stripping is an addition, not a replacement: the written label wins.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual( + [item.id for item in graph.seeds("parse_graph_citation()")], ["n-citation"]) + + def test_an_exact_label_beats_the_same_string_read_as_a_callable(self) -> None: + """Both exist, and the tier that matched what was written is the answer. + + A module attribute ``packet`` and a function ``packet()`` are different + definitions. Merging them would report a relationship of one as a + relationship of the other, so the exact label resolves alone and the + result is not ambiguous. + """ + graph = self.load( + node("n-attribute", "packet", "example_pkg/config.py", 4), + node("n-function", "packet()", "example_pkg/config.py", 12), + ) + self.assertEqual([item.id for item in graph.seeds("packet")], ["n-attribute"]) + + def test_a_near_name_does_not_reach_a_longer_callable(self) -> None: + """No prefix, substring, or edit-distance matching -- an equality test.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12), + node("n-other", "parse_graph_citations()", "example_pkg/config.py", 20), + ) + for target in ("parse_graph", "parse", "graph_citation", "arse_graph_citation"): + with self.subTest(target=target): + self.assertEqual(graph.seeds(target), ()) + + def test_overload_like_labels_are_ambiguity_rather_than_a_pick(self) -> None: + """Two labels reduce to one name, so the target names two definitions.""" + graph = self.load( + node("n-int", "render(int)", "example_pkg/report.py", 5), + node("n-str", "render(str)", "example_pkg/report.py", 9), + ) + result = query.run_query(graph, question="symbol", target="render") + self.assertEqual([item.id for item in result.seeds], ["n-int", "n-str"]) + self.assertTrue(result.ambiguous) + self.assertIn("unresolved_entities", result.omissions) + + def test_the_seed_bound_and_its_truncation_still_apply_to_bare_names(self) -> None: + nodes = [ + node(f"n-{index:02d}", f"render(arg{index})", "example_pkg/report.py", index + 1) + for index in range(query.MAX_SEEDS + 1) + ] + result = query.run_query(self.load(*nodes), question="symbol", target="render") + self.assertEqual(len(result.seeds), query.MAX_SEEDS) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_a_path_target_is_unchanged_and_still_last(self) -> None: + """Paths are literal, and a callable label never stands in for one.""" + graph = self.load( + node("n-citation", "parse_graph_citation()", "example_pkg/config.py", 12)) + self.assertEqual( + [item.id for item in graph.seeds("example_pkg/config.py")], ["n-citation"]) + self.assertEqual(graph.seeds("example_pkg"), ()) + + def test_only_a_whole_trailing_argument_list_is_removed(self) -> None: + """The rule as a table, including every shape that must reduce to nothing.""" + for label, expected in ( + ("parse_graph_citation()", "parse_graph_citation"), + ("render(int)", "render"), + ("render(Callable[(int)])", "render"), + ("parse_graph_citation", ""), + ("()", ""), + ("render(int))", ""), + ("render()x", ""), + ("render(int", ""), + ("ren)der()", ""), + ("", ""), + ): + with self.subTest(label=label): + self.assertEqual(query._callable_base(label), expected) + + class CitationValidationTests(GraphWorkspace): def validator(self) -> query.CitationValidator: census = lifecycle.read_tracked_census(self.repository, self.manifest.commit) From 85f966ad547faeb431c255dfb6b96dfb47739487 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 09:04:43 -0700 Subject: [PATCH 14/33] Graph lifecycle: classify inputs the way the pin does, and report what it could not extract The completeness denominator was suffix membership alone, which is not how the pinned detect.classify_file decides. It routes named package manifests by filename first -- apm.yml, apm.yaml, pyproject.toml, Cargo.toml, go.mod, pom.xml -- and extensionless shebang scripts second, reaching the extension table only after both. None of those manifest names carries a code extension, so a manifest the provider failed on could drop out of the coverage question entirely while an unrelated .py file let the run claim it was complete. _provider_inputs now classifies the census in the pin's own order, from the staged exact 0.9.58 registry, and reads the shebang of an extensionless input from the immutable materialized copy before the launch. An input it cannot decide -- unreadable bytes, or one of the env(1) option spellings the pin resolves through a parser this adapter deliberately does not reimplement -- is carried as unclassified and keeps the run partial, because neither guess is safe: "not code" drops a real input from the denominator and "code" demands a row for a file that was never dispatched. A stamped row is now read for what the pin's post-extraction writer rule actually makes it mean. The CLI clears exactly the rows in _failed_sources and stamps every other dispatched input; a result is a failed source when it carries an error or when its extractor produced zero nodes, and it is not when _get_extractor returned None or when the extractor declined by design, as the JSON extractor does for data JSON and non-object roots. So a matching hash proves read bytes and no failure, not a supported extraction. The deterministically unsupported dispatch -- the exact set difference of the pin's CODE_EXTENSIONS and _DISPATCH, plus the shebang interpreters detect treats as code that _SHEBANG_DISPATCH does not map -- is counted separately and reported rather than folded into indexed_files. That count survives the build: IndexResult.unsupported_inputs reaches the manifest, round-trips through validation, and appears in the summary status emits and in the operator-facing text. It stays distinct from skipped_paths, which counts what the build declined to materialize at all; the two answer different questions and their sum answers neither. load_manifest accepts the field as optional so a generation published before it existed still loads as zero, and widens by exactly that one name -- any other unrecognized key is still a refusal. The docstring and docs claim that the evaluation's 54 requeued entries were blank-hash rows is corrected: the retained evidence for that run carries stamped rows, so requeueing there is not observable as a blank row and nothing here claims it is. The requeue behaviour stays recorded as an observed limitation of the provider's incremental gate without inventing a report shape for it. The CommandTests round-trip stand-in stamped a constant hash for every input, which no longer satisfies the hash comparison it is supposed to exercise. It now stamps the real digest of the bytes it was shown, so a build/refresh round trip meets the same contract a real provider has to meet, and the strict validation is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-lifecycle.md | 52 +++- src/code_mower/context_graph_lifecycle.py | 353 +++++++++++++++++++--- tests/test_context_graph_lifecycle.py | 173 ++++++++++- 3 files changed, 518 insertions(+), 60 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index 3e751f60..d44fcebd 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -324,6 +324,7 @@ Every published generation carries, in `manifest.json`: | `graph_digest`, `graph_bytes` | Detects a truncated or tampered artifact on every read. | | `completeness` | `complete` or `partial`, from the provider's own admission. | | `indexed_files` | What the provider claims it processed, bounded by the census. | +| `unsupported_inputs` | How many inputs the provider classified as code and then deterministically could not extract — no wired extractor, or an extractor that declined by design. Read bytes, no contribution. Kept separate from `skipped_paths`, which counts what this build declined to materialize at all; the two answer different questions and their sum answers neither. Optional on read so a generation published before the field existed still loads as `0`; always written. | `shareable_summary()` is the metadata-only view: revisions, digests, counts and states. It carries no indexed content, no provider output, and no local path. @@ -410,22 +411,51 @@ what says it is a hash of this input, and without it a manifest carried over from another tree, another revision, or a resumed cache would read as proof of work on bytes the provider never saw. +The denominator is the census classified the way the pinned +`detect.classify_file` classifies it, **in its order** — not by extension +membership. A package manifest is routed by filename first (`apm.yml`, +`apm.yaml`, `pyproject.toml`, `Cargo.toml`, `go.mod`, `pom.xml`), an +extensionless file by its shebang second, and only then is the extension table +reached. Suffix membership alone misses both: `.yml`, `.toml`, `.mod` and +`.xml` are not code extensions, so a `pyproject.toml` the provider failed on +could drop out of the coverage question entirely while an unrelated `.py` file +let the run claim it was complete. + Everything else is `partial`: a row the manifest never wrote, a blank or malformed hash, a hash that disagrees with those bytes, an input the copy could -not be re-read for, a record this adapter cannot classify, an unreadable or -oversized manifest, no manifest at all, and a build with no census. The pin -blanks hashes through `clear_ast` on an extractor error or an anomalous -zero-node extract, so a requeued file is a blank row rather than an absent one -— the direct consequence of the requeue defect the evaluation recorded, where a -repeat that exits zero in 1.63 seconds having requeued 54 entries has not built -a complete graph. +not be re-read for, a record this adapter cannot classify, an input it cannot +classify against the provider's own dispatch, an unreadable or oversized +manifest, no manifest at all, and a build with no census. + +### What a stamped row is evidence of + +From the pin's own post-extraction writer rule. After a run the CLI clears +(`clear_ast`) exactly the rows in `_failed_sources`, and stamps every other +dispatched input. A result is a failed source when it carries an `error`, or +when its extractor produced zero nodes. It is **not** when `_get_extractor` +returned `None` — the file short-circuits to an empty result with neither +marker — and **not** when the extractor declined by design, as the JSON +extractor does for data JSON and for a non-object root. + +So a stamped, matching row proves the provider read those exact bytes and did +not fail on them. It does not prove nodes. The deterministically unsupported +dispatch is therefore counted and reported on its own line +(`unsupported_inputs`) rather than folded into `indexed_files`, and zero nodes +for a stamped file is a complete *read* of that file and nothing more. + +Blank rows are the cases the pin's rule makes blank: an extractor error or an +anomalous zero-node extract. They stay `partial` here. The evaluation's +clean-room repeat that exited zero in 1.63 seconds requeued 54 entries; the +retained evidence for that run carries *stamped* rows, so requeueing there is +not observable as a blank row, and nothing in this module claims it is. That +requeue behaviour remains an observed limitation of the provider's incremental +gate rather than a shape this adapter reports on. Nothing upgrades a run. The exit status, a non-empty graph, and the raw extraction's `extracted_sources` all describe what was *dispatched*, failures -included. A hash proves processed bytes, not that anything was understood: zero -nodes for a stamped file is still a complete read of it, and an unstamped file -is `partial` however large the graph is. `partial` is the state `graph_status` -refuses by default, so the failure is one an operator can see and act on. +included. An unstamped file is `partial` however large the graph is. `partial` +is the state `graph_status` refuses by default, so the failure is one an +operator can see and act on. The manifest is provider output of unknown size, so it is read to one byte past the manifest bound and refused if it is longer, rather than loaded whole and diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index 4557a1ef..b19c94a1 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -58,6 +58,7 @@ import re import io import secrets +import shlex import shutil import signal import socket @@ -1363,6 +1364,12 @@ class IndexResult: completeness: str = COMPLETE indexed_files: int = 0 + #: Inputs this provider classified as code and then deterministically could + #: not extract -- no wired extractor, or an extractor that declined by + #: design. Read bytes, no contribution; reported rather than counted as + #: indexed, and distinct from the census entries a build declines to + #: materialize at all. + unsupported_inputs: int = 0 notes: tuple[str, ...] = () @@ -2251,6 +2258,58 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: ".robot", ".resource", }) +#: Names the pinned ``detect.classify_file`` routes to code by *filename*, +#: ahead of every suffix class, because ``manifest_ingest`` parses them +#: deterministically (``PACKAGE_MANIFEST_NAMES``, compared lower-cased against +#: the basename). Suffix membership alone misses all of them -- ``.yml``, +#: ``.toml``, ``.mod`` and ``.xml`` are not in ``CODE_EXTENSIONS`` -- so a +#: denominator built from extensions would let a package manifest the provider +#: failed on drop out of the coverage question entirely. +_PROVIDER_PACKAGE_MANIFEST_NAMES = frozenset({ + "apm.yml", "apm.yaml", "pyproject.toml", "cargo.toml", "go.mod", "pom.xml", +}) + +#: The one compound suffix the pin tests before the simple one. ``.blade.php`` +#: already ends in a code extension, so this only decides *which* extractor the +#: pin uses; it is named here because the classification below is meant to be +#: readable against ``classify_file``'s own order rather than to be minimal. +_PROVIDER_COMPOUND_CODE_SUFFIX = ".blade.php" + +#: ``detect._SHEBANG_CODE_INTERPRETERS``: the interpreters that make an +#: *extensionless* tracked file code to this pin. ``classify_file`` reaches +#: this branch before any extension test, so a CLI entry point with no suffix +#: is dispatched exactly like a ``.py`` file and belongs in the denominator. +_PROVIDER_SHEBANG_CODE_INTERPRETERS = frozenset({ + "python", "python3", "python2", + "ruby", "perl", "node", "nodejs", + "bash", "sh", "dash", "zsh", "fish", "ksh", "tcsh", + "lua", "php", "julia", "Rscript", +}) + +#: ``extract._SHEBANG_DISPATCH``: the subset of the above that the pin has an +#: extractor for. The remainder (``perl``, ``fish``, ``tcsh``, ``Rscript``) is +#: classified as code and then deterministically contributes nothing. +_PROVIDER_SHEBANG_EXTRACTORS = frozenset({ + "python", "python2", "python3", + "bash", "sh", "dash", "zsh", "ksh", + "node", "nodejs", "ruby", "lua", "php", "julia", +}) + +#: The extensions in the pin's ``CODE_EXTENSIONS`` with no entry in +#: ``extract._DISPATCH`` -- the exact set difference of the two staged tables. +#: The pin warns about these itself (#1689: "classified as code but graphify +#: has no AST extractor for their language"), dispatches them, and stamps them, +#: because ``_get_extractor`` returning ``None`` short-circuits to +#: ``{"nodes": [], "edges": []}`` with neither an ``error`` nor a ``skipped`` +#: marker, and the CLI's failed-source rule only clears rows for the two cases +#: that carry one. They are counted and reported, never silently folded into +#: the files this build says were indexed. +_PROVIDER_UNSUPPORTED_EXTENSIONS = frozenset({".ejs", ".ets", ".r"}) + +#: How much of an extensionless input is read to find its shebang. The pin +#: reads the same 256 bytes and keeps only the first line. +_SHEBANG_PROBE_BYTES = 256 + #: A manifest row's content hash, as the pin computes it: ``_md5_file`` streams #: the file and returns a hex digest, or the empty string when the read failed. #: So a well-formed 32-character digest is the provider's own statement that it @@ -2298,7 +2357,9 @@ def _materialized_digest(path: Path) -> str: def _materialized_digests( - source_root: Path, census: TrackedCensus | None + source_root: Path, + census: TrackedCensus | None, + inputs: _ProviderInputs | None = None, ) -> dict[str, str] | None: """What every eligible code input's manifest row must say, before the run. @@ -2315,16 +2376,14 @@ def _materialized_digests( """ if census is None: return None + if inputs is None: + inputs = _provider_inputs(census, source_root) digests: dict[str, str] = {} - for path in _eligible_code_inputs(census): - parts = PurePosixPath(path).parts - # ``read_tracked_census`` reads Git's own index paths, which are - # relative and carry no traversal segment. Held to that here anyway, - # because this is the one place a census path is turned back into a - # host path, and a join is not the place to discover otherwise. - if not parts or any(part in ("", ".", "..", "/") for part in parts): + for path in inputs.dispatched: + host = _census_host_path(source_root, path) + if host is None: continue - digest = _materialized_digest(source_root.joinpath(*parts)) + digest = _materialized_digest(host) if digest: digests[unicodedata.normalize("NFC", path)] = digest return digests @@ -2411,26 +2470,146 @@ def _provider_manifest(output_directory: Path) -> Mapping[str, Any] | None: return payload if isinstance(payload, Mapping) else None -def _eligible_code_inputs(census: TrackedCensus) -> tuple[str, ...]: - """The census paths this pin would dispatch, in census order. +@dataclass(frozen=True) +class _ProviderInputs: + """How the pin would classify this census, decided before it is launched. + + ``dispatched`` is the denominator: every tracked path ``classify_file`` + would call code, in census order. ``unsupported`` is the subset of those the + pin then has no extractor for, which is a real and reportable outcome rather + than a failure. ``unclassified`` is everything the classification could not + decide -- an extensionless input the copy could not be read for, or a + shebang spelling this adapter refuses to guess at -- and it keeps the run + partial, because an input nobody can classify is an input nobody can say was + covered. + """ + + dispatched: tuple[str, ...] = () + unsupported: frozenset[str] = frozenset() + unclassified: tuple[str, ...] = () + + +def _census_host_path(source_root: Path, path: str) -> Path | None: + """Where a census path lives in the materialized copy, or ``None``. + + ``read_tracked_census`` reads Git's own index paths, which are relative and + carry no traversal segment. Held to that here anyway, because this is the + only place a census path is turned back into a host path, and a join is not + the place to discover otherwise. + """ + parts = PurePosixPath(path).parts + if not parts or any(part in ("", ".", "..", "/") for part in parts): + return None + return source_root.joinpath(*parts) + + +def _shebang_interpreter(path: Path) -> tuple[str | None, bool]: + """``(interpreter, resolved)`` for an extensionless input's first line. - Case is tried both ways because the pin's set carries both ``.f90`` and - ``.F90``: matching the spelling first and the lower-cased suffix second can - only widen the denominator, which is the direction that refuses rather than - over-claims. + ``(None, True)`` is a decision: there is no shebang, so ``classify_file`` + would not call this file code. ``(None, False)`` is a refusal: the bytes + could not be read, or the line is one of the ``env(1)`` spellings the pin + resolves through option parsing this adapter deliberately does not + reimplement (``-S``/``--split-string`` and friends). A refusal is carried as + *unclassified* rather than guessed either way, because guessing "not code" + would drop a real input out of the denominator and guessing "code" would + demand a row for a file the pin never dispatched. + + Only the simple, unambiguous ``env`` forms are resolved here: leading + ``NAME=value`` assignments followed by the interpreter, which is what a + tracked script ordinarily carries. """ - eligible: list[str] = [] + try: + if path.is_symlink() or not path.is_file(): + return None, False + with path.open("rb") as stream: + head = stream.read(_SHEBANG_PROBE_BYTES) + except OSError: + return None, False + if not head.startswith(b"#!"): + return None, True + line = head.split(b"\n")[0].decode(errors="replace")[2:].strip() + try: + parts = shlex.split(line) + except ValueError: + return None, False + if not parts: + return None, True + interpreter = PurePosixPath(parts[0].replace("\\", "/")).name + if interpreter != "env": + return interpreter, True + for argument in parts[1:]: + if argument.startswith("-"): + # An option-carrying ``env`` line. The pin has a full parser for + # these; this one says so rather than pretending to. + return None, False + if "=" in argument: + continue + return PurePosixPath(argument.replace("\\", "/")).name, True + return None, True + + +def _provider_inputs( + census: TrackedCensus, source_root: Path | None = None +) -> _ProviderInputs: + """Classify the census the way the pinned ``detect.classify_file`` would. + + In its order, which is the part suffix membership gets wrong: a package + manifest is routed by *filename* before any extension is looked at, and an + extensionless file is routed by its *shebang* before the extension table is + reached at all. A denominator built from ``CODE_EXTENSIONS`` alone drops + both, so a ``pyproject.toml`` or a ``#!/usr/bin/env python3`` CLI the + provider failed on could be missing from the manifest while an unrelated + ``.py`` file let the run claim it was complete. + + Case is tried both ways for the extension test because the pin's set carries + both ``.f90`` and ``.F90``; matching the spelling first and the lower-cased + suffix second can only widen the denominator, which is the direction that + refuses rather than over-claims. + + ``source_root`` is the materialized copy, read *before* the provider is + launched -- the only moment those bytes are still exactly what this build + handed over. Without it no extensionless input can be classified, so every + one of them is unclassified and the run stays partial. + """ + dispatched: list[str] = [] + unsupported: set[str] = set() + unclassified: list[str] = [] for entry in census.entries: - suffix = PurePosixPath(entry.path).suffix + path = entry.path + pure = PurePosixPath(path) + name = pure.name.lower() + if name in _PROVIDER_PACKAGE_MANIFEST_NAMES or name.endswith( + _PROVIDER_COMPOUND_CODE_SUFFIX + ): + dispatched.append(path) + continue + suffix = pure.suffix + if not suffix: + host = _census_host_path(source_root, path) if source_root is not None else None + if host is None: + unclassified.append(path) + continue + interpreter, resolved = _shebang_interpreter(host) + if not resolved: + unclassified.append(path) + elif interpreter in _PROVIDER_SHEBANG_CODE_INTERPRETERS: + dispatched.append(path) + if interpreter not in _PROVIDER_SHEBANG_EXTRACTORS: + unsupported.add(path) + continue if suffix in _PROVIDER_CODE_EXTENSIONS or suffix.lower() in _PROVIDER_CODE_EXTENSIONS: - eligible.append(entry.path) - return tuple(eligible) + dispatched.append(path) + if suffix.lower() in _PROVIDER_UNSUPPORTED_EXTENSIONS: + unsupported.add(path) + return _ProviderInputs(tuple(dispatched), frozenset(unsupported), tuple(unclassified)) def _read_completeness( manifest: Mapping[str, Any] | None, census: TrackedCensus | None, digests: Mapping[str, str] | None = None, + inputs: _ProviderInputs | None = None, ) -> IndexResult: """Classify a provider run against its own manifest, defaulting to partial. @@ -2440,9 +2619,12 @@ def _read_completeness( coverage question: did every input this pin would dispatch come back with a hash proving the provider read its bytes? - The denominator is the immutable materialized census narrowed to the pin's - own code extensions. Inputs outside that set are deterministically not code - to this pin and are skipped, not missing. Everything else is counted, and a + The denominator is the immutable materialized census classified the way + ``detect.classify_file`` classifies it -- filename-routed package manifests + first, then extensionless shebang scripts, then the extension table (see + ``_provider_inputs``). Inputs outside that classification are + deterministically not code to this pin and are skipped, not missing. + Everything else is counted, and a file is processed only when its row carries a well-formed ``ast_hash`` *and that hash is the digest of the bytes this build actually handed the provider*. A well-formed digest alone says a hash-shaped string is present; @@ -2451,25 +2633,47 @@ def _read_completeness( as proof of work on bytes the provider was never shown -- and the digests come from ``_materialized_digests``, taken before the launch, so they cannot have been influenced by what the run wrote. A row whose digest disagrees, and - an input the copy could not be re-read for, both stay partial. The - pin blanks that field on exactly the cases an operator needs to hear about: - ``clear_ast`` zeroes both hashes for an extractor error or an anomalous - zero-node extract, so a requeued file is a blank row rather than an absent - one. The clean-room run's 54 requeued entries -- from a repeat that exited - zero in 1.63 s -- are that shape, and they stay partial here. + an input the copy could not be re-read for, both stay partial. + + What a stamped row is evidence *of* comes from the pin's own + post-extraction writer rule, staged in ``extract.py`` and ``cli.py``. After + the run, ``_failed_sources`` is assembled from the per-file results and the + CLI clears (``clear_ast``) exactly those rows; every other dispatched input + is stamped. A result lands in ``_failed_sources`` when it carries an + ``error``, or when its extractor produced zero nodes. It does **not** when: + + * ``_get_extractor`` returned ``None`` -- the file short-circuits to + ``{"nodes": [], "edges": []}`` with neither marker, so a code-classified + input the pin has no extractor for is stamped while contributing nothing + (the pin's own #1689 warning); or + * the extractor declined by design -- ``extractors/json_config`` returns a + ``skipped`` marker for data JSON and for a non-object root, and the CLI + skips those deliberately so they are not requeued forever (#2879). + + So a stamped, matching row proves the provider read those exact bytes and + did not fail on them. It does not prove nodes, and it is not read here as + if it did. The deterministically unsupported dispatch is counted and + reported separately (``unsupported_inputs``) rather than folded into + ``indexed_files``, and zero nodes for a file whose row is stamped is a + complete *read* of that file and nothing more. + + Blank rows are the cases the pin's rule makes blank: an extractor error or + an anomalous zero-node extract. They stay partial here. The clean-room + repeat that exited zero in 1.63 s requeued 54 entries; the retained + evidence for that run carries *stamped* rows, so requeueing there is not + observable as a blank row and nothing in this adapter claims it is. That + remains an observed limitation of the incremental gate rather than a shape + this module reports on. Nothing upgrades a run: the exit status, a non-empty graph, and the raw extraction's ``extracted_sources`` are all statements about what was - *dispatched*, failures included, so none of them is success evidence. A - hash proves processed bytes, not that anything was understood; zero nodes - for a file whose row is stamped is still a complete read of that file, and - an unstamped one is partial however large the graph is. Missing evidence, - an unparseable manifest, a shape this adapter does not recognize, and a row - that cannot be told apart from a failure all stay ``partial``, which - ``graph_status`` refuses by default. The provider owns no provenance (the - evaluation records this as the first product constraint), so that refusal - is the failure an operator can act on; silently calling it complete is the - one they cannot. + *dispatched*, failures included, so none of them is success evidence. + Missing evidence, an unparseable manifest, a shape this adapter does not + recognize, an input it could not classify, and a row that cannot be told + apart from a failure all stay ``partial``, which ``graph_status`` refuses + by default. The provider owns no provenance (the evaluation records this as + the first product constraint), so that refusal is the failure an operator + can act on; silently calling it complete is the one they cannot. """ if census is None: return IndexResult( @@ -2486,7 +2690,9 @@ def _read_completeness( completeness=PARTIAL, notes=("build recorded no input digests to check the provider's manifest against",), ) - eligible = _eligible_code_inputs(census) + if inputs is None: + inputs = _provider_inputs(census) + eligible = inputs.dispatched rows: dict[str, Any] = {} malformed_rows = 0 for key, row in manifest.items(): @@ -2499,11 +2705,15 @@ def _read_completeness( malformed_rows += 1 continue rows[unicodedata.normalize("NFC", key)] = row + unsupported_keys = { + unicodedata.normalize("NFC", path) for path in inputs.unsupported + } missing = 0 unstamped = 0 unreadable = 0 mismatched = 0 processed = 0 + unsupported = 0 for path in eligible: key = unicodedata.normalize("NFC", path) row = rows.get(key) @@ -2522,6 +2732,11 @@ def _read_completeness( unreadable += 1 elif digest != expected: mismatched += 1 + elif key in unsupported_keys: + # Read, not failed, and deterministically not extractable by this + # pin. Counted on its own line rather than as a file this build + # indexed, which it is not. + unsupported += 1 else: processed += 1 notes: list[str] = [] @@ -2535,11 +2750,23 @@ def _read_completeness( notes.append(f"build could not re-read {unreadable} code files to check their hashes") if mismatched: notes.append(f"provider hashed {mismatched} code files that are not the bytes it was given") + if inputs.unclassified: + notes.append( + f"build could not classify {len(inputs.unclassified)} tracked inputs " + "against this provider's own dispatch" + ) if not eligible: notes.append("the census carried no code files this provider would index") if notes: - return IndexResult(completeness=PARTIAL, indexed_files=processed, notes=tuple(notes)) - return IndexResult(completeness=COMPLETE, indexed_files=processed) + return IndexResult( + completeness=PARTIAL, + indexed_files=processed, + unsupported_inputs=unsupported, + notes=tuple(notes), + ) + return IndexResult( + completeness=COMPLETE, indexed_files=processed, unsupported_inputs=unsupported + ) class _BoundedBuffer(io.BytesIO): @@ -2854,7 +3081,15 @@ def run(request: IndexRequest) -> IndexResult: # the provider; once the child has run, the same tree also holds the # provider's own output, and a digest taken then would be checking the # provider's manifest against the provider's own leavings. - digests = _materialized_digests(request.source_root, request.census) + # Classified and digested from the copy *before* the launch: after the + # run the same tree also holds provider output, and a shebang read then + # would be classifying whatever the provider left behind. + inputs = ( + _provider_inputs(request.census, request.source_root) + if request.census is not None + else None + ) + digests = _materialized_digests(request.source_root, request.census, inputs) # Built per run, because the boundary is a function of what this build # exposes: the materialized copy and the build's own scratch areas are # writable, the pinned provider's install is readable, and nothing else @@ -2888,7 +3123,7 @@ def run(request: IndexRequest) -> IndexResult: raise ContextError("local graph provider failed; no generation was published") output_directory = _provider_output_directory(request.source_root) result = _read_completeness( - _provider_manifest(output_directory), request.census, digests + _provider_manifest(output_directory), request.census, digests, inputs ) _write_private_file(request.output_path, _pack_state(output_directory)) return result @@ -2921,6 +3156,13 @@ class BuildManifest: completeness: str skipped_paths: int indexed_files: int + #: Inputs the provider classified as code and deterministically could not + #: extract. Deliberately *not* folded into ``skipped_paths``, which counts + #: tracked entries this build declined to materialize at all (symlinks, + #: submodules, private state). Those two numbers answer different questions + #: -- what this build withheld, and what the provider could not read -- + #: and an operator who needs to act on one cannot act on their sum. + unsupported_inputs: int = 0 def to_json(self) -> dict[str, Any]: return { @@ -2938,6 +3180,7 @@ def to_json(self) -> dict[str, Any]: "completeness": self.completeness, "skipped_paths": self.skipped_paths, "indexed_files": self.indexed_files, + "unsupported_inputs": self.unsupported_inputs, } def shareable_summary(self) -> dict[str, Any]: @@ -2954,11 +3197,21 @@ def shareable_summary(self) -> dict[str, Any]: "graph_digest": self.graph_digest, "graph_bytes": self.graph_bytes, "completeness": self.completeness, + "unsupported_inputs": self.unsupported_inputs, } def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: - """Validate a manifest. Every unreadable shape is a refusal, not a default.""" + """Validate a manifest. Every unreadable shape is a refusal, not a default. + + ``unsupported_inputs`` is accepted as optional so a generation published + before it existed still loads. It is always written, so the only manifests + that take the default are older ones, and ``0`` is the honest reading of + them: that build never counted the provider's unsupported dispatch, and a + zero says the same thing a missing key does. Nothing else is optional -- + an unrecognized key is still a refusal, so this widens what loads by + exactly one name. + """ if not isinstance(payload, Mapping): raise ContextError("local graph manifest must be an object") expected = { @@ -2966,7 +3219,9 @@ def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: "tracked_files", "tracked_bytes", "census_digest", "graph_digest", "graph_bytes", "completeness", "skipped_paths", "indexed_files", } - if set(payload) != expected: + optional = {"unsupported_inputs"} + present = set(payload) + if not expected <= present or not present <= (expected | optional): raise ContextError("local graph manifest fields are missing or unrecognized") if payload["schema"] != MANIFEST_SCHEMA: raise ContextError("unsupported local graph manifest schema") @@ -2996,6 +3251,9 @@ def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: completeness=completeness, skipped_paths=_size(payload["skipped_paths"], MAX_SKIPPED_PATHS), indexed_files=_size(payload["indexed_files"], MAX_TRACKED_FILES), + unsupported_inputs=_size( + payload.get("unsupported_inputs", 0), MAX_TRACKED_FILES + ), ) @@ -3782,6 +4040,9 @@ def build_graph( completeness=result.completeness, skipped_paths=len(census.skipped), indexed_files=min(_size(result.indexed_files, MAX_TRACKED_FILES), census.file_count), + unsupported_inputs=min( + _size(result.unsupported_inputs, MAX_TRACKED_FILES), census.file_count + ), ) published = state.publish(manifest, artifact) if not keep_previous: @@ -3998,6 +4259,8 @@ def render_status_text(status: GenerationStatus) -> str: f" census: {manifest.census_digest}", f" graph: {manifest.graph_digest} ({manifest.graph_bytes} bytes)", f" complete: {manifest.completeness}", + f" indexed: {manifest.indexed_files} files " + f"({manifest.unsupported_inputs} unsupported by this provider)", ] ) return "\n".join(lines) + "\n" diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index 75d4c213..e5b125f1 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -130,14 +130,19 @@ def kill(self) -> None: def recording_indexer(payload: bytes = b"graph-bytes", *, completeness: str = lifecycle.COMPLETE, - seen: list | None = None, indexed_files: int = 0): + seen: list | None = None, indexed_files: int = 0, + unsupported_inputs: int = 0): """An indexer that writes a fixed artifact and records what it was shown.""" def run(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: if seen is not None: seen.append(request) request.output_path.write_bytes(payload) - return lifecycle.IndexResult(completeness=completeness, indexed_files=indexed_files) + return lifecycle.IndexResult( + completeness=completeness, + indexed_files=indexed_files, + unsupported_inputs=unsupported_inputs, + ) return run @@ -1544,6 +1549,105 @@ def test_a_code_input_the_copy_does_not_hold_cannot_be_counted(self) -> None: self.assertIn("could not re-read 1 code files", " ".join(result.notes)) self.assertEqual(result.indexed_files, 0) + def test_a_named_package_manifest_is_in_the_denominator(self) -> None: + """``classify_file`` routes these by filename before any suffix class. + + ``.toml`` is not a code extension, so an extension-only denominator + drops ``pyproject.toml`` entirely: the provider could fail on it while + ``src/app.py`` alone let the run claim it was complete. Every name the + pin's ``PACKAGE_MANIFEST_NAMES`` carries is checked, including the + mixed-case spellings the repositories that use them actually commit. + """ + for name in ( + "pyproject.toml", "Cargo.toml", "go.mod", "pom.xml", "apm.yml", "apm.yaml", + ): + with self.subTest(name=name): + inputs = lifecycle._provider_inputs(census_of(CODE_INPUT, name)) + self.assertIn(name, inputs.dispatched) + self.assertEqual(inputs.unsupported, frozenset()) + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census_of(CODE_INPUT, name), + {CODE_INPUT: CODE_INPUT_DIGEST}, + inputs, + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) + + def test_a_shebang_script_without_a_suffix_is_in_the_denominator(self) -> None: + """The pin routes extensionless files by their first line, so this does. + + Both halves matter: a supported interpreter is a dispatched input whose + absence from the manifest keeps the run partial, and one the pin has no + extractor for is dispatched *and* reported as unsupported rather than + counted as work. + """ + root = self.root / "copy" + (root / "bin").mkdir(parents=True) + (root / "bin" / "devctl").write_bytes(b"#!/usr/bin/env python3\nprint(1)\n") + (root / "bin" / "report").write_bytes(b"#!/usr/bin/perl\nprint 1;\n") + (root / "bin" / "notes").write_bytes(b"plain text, no shebang\n") + (root / "bin" / "packed").write_bytes(b"#!/usr/bin/env -S python3 -u\n") + inputs = lifecycle._provider_inputs( + census_of("bin/devctl", "bin/report", "bin/notes", "bin/packed"), root + ) + self.assertEqual( + set(inputs.dispatched), {"bin/devctl", "bin/report"} + ) + # perl is code to ``detect`` and has no entry in ``_SHEBANG_DISPATCH``. + self.assertEqual(inputs.unsupported, frozenset({"bin/report"})) + # No shebang is a decision, not a refusal: the pin would not call it code. + self.assertNotIn("bin/notes", inputs.unclassified) + # ``env -S`` is the option-carrying form this adapter declines to guess at. + self.assertEqual(inputs.unclassified, ("bin/packed",)) + + def test_an_input_that_cannot_be_classified_keeps_the_run_partial(self) -> None: + """Unknown is not the same as not-code, and must not read as coverage.""" + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census_of(CODE_INPUT), + {CODE_INPUT: CODE_INPUT_DIGEST}, + lifecycle._ProviderInputs( + dispatched=(CODE_INPUT,), unclassified=("bin/packed",) + ), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not classify 1 tracked inputs", " ".join(result.notes)) + + def test_unsupported_dispatch_is_counted_apart_from_indexed_files(self) -> None: + """A stamped row proves read bytes, not a supported extraction. + + ``_get_extractor`` returns ``None`` for a code-classified extension with + no wired extractor, which short-circuits to an empty result carrying + neither an ``error`` nor a ``skipped`` marker -- so the CLI's + failed-source rule leaves the row stamped. Counting that as an indexed + file would report work the provider demonstrably did not do. + """ + census = census_of(CODE_INPUT, "analysis/model.r") + digest = hashlib.md5(b"x", usedforsecurity=False).hexdigest() + result = lifecycle._read_completeness( + {CODE_INPUT: manifest_row(), "analysis/model.r": manifest_row(digest)}, + census, + {CODE_INPUT: CODE_INPUT_DIGEST, "analysis/model.r": digest}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + self.assertEqual(result.unsupported_inputs, 1) + + def test_an_unsupported_extension_still_has_to_be_accounted_for(self) -> None: + """Reported, not excused: an absent row for one is still partial.""" + census = census_of(CODE_INPUT, "web/view.ejs") + result = lifecycle._read_completeness( + FINISHED_MANIFEST, + census, + {CODE_INPUT: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("does not account for 1 code files", " ".join(result.notes)) + self.assertEqual(result.unsupported_inputs, 0) + def test_a_request_without_a_census_cannot_be_complete(self) -> None: """There is no denominator, so there is no coverage claim to make.""" self.assertEqual( @@ -2836,6 +2940,48 @@ def test_manifest_round_trips_through_validation(self) -> None: manifest = self.build() self.assertEqual(lifecycle.load_manifest(manifest.to_json()), manifest) + def test_the_unsupported_count_survives_the_build_and_the_status_report(self) -> None: + """The number has to outlive the indexer, or it is not a report. + + ``IndexResult.notes`` are read by the build and then dropped, so a + count that lived only there would never reach an operator. This one is + written into the manifest, round-trips through validation, and appears + in the summary ``status`` emits. + """ + manifest = self.build( + indexer=recording_indexer(indexed_files=1, unsupported_inputs=2) + ) + self.assertEqual(manifest.unsupported_inputs, 2) + self.assertEqual(manifest.to_json()["unsupported_inputs"], 2) + self.assertEqual(lifecycle.load_manifest(manifest.to_json()), manifest) + self.assertEqual(manifest.shareable_summary()["unsupported_inputs"], 2) + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertIsNotNone(status.manifest) + self.assertEqual(status.shareable_summary()["build"]["unsupported_inputs"], 2) + + def test_the_unsupported_count_is_not_the_unmaterialized_count(self) -> None: + """Two different questions, so two different numbers. + + ``skipped_paths`` is what this build declined to materialize; the new + count is what the provider could not extract from what it *was* given. + Folding either into the other would report a number an operator cannot + act on. + """ + manifest = self.build( + indexer=recording_indexer(indexed_files=1, unsupported_inputs=1) + ) + self.assertEqual(manifest.skipped_paths, 0) + self.assertEqual(manifest.unsupported_inputs, 1) + + def test_a_manifest_published_before_the_count_existed_still_loads(self) -> None: + """Compatibility, bounded to exactly the one new name.""" + payload = self.build().to_json() + payload.pop("unsupported_inputs") + self.assertEqual(lifecycle.load_manifest(payload).unsupported_inputs, 0) + payload["some_other_field"] = 1 + with self.assertRaises(ContextError): + lifecycle.load_manifest(payload) + def test_shareable_summary_carries_no_content_or_local_path(self) -> None: summary = self.build().shareable_summary() rendered = json.dumps(summary) @@ -3636,12 +3782,31 @@ def indexer_script(self, *, complete: bool = True, **installed) -> Path: # file looks like to the collector. + ( ( + # The real pin stamps ``_md5_file`` of the bytes it was + # handed, and the collector checks the row against the + # immutable materialized copy. A constant here would be + # a stand-in that never reads its inputs, so it stamps + # the actual digest -- ``md5 -q`` on macOS, ``md5sum`` + # elsewhere -- and a build/refresh round trip exercises + # the same contract a real provider has to meet. + # Three spellings because the host decides which exists: + # openssl and md5 ship with macOS, md5sum with GNU + # coreutils. An empty result is left empty rather than + # faked, so a host with none of them fails the coverage + # check loudly instead of passing on a constant. + "digest_of() {\n" + " h=$(openssl dgst -md5 -r \"$1\" 2>/dev/null | cut -d' ' -f1)\n" + " [ -n \"$h\" ] || h=$(md5 -q \"$1\" 2>/dev/null)\n" + " [ -n \"$h\" ] || h=$(md5sum \"$1\" 2>/dev/null | cut -d' ' -f1)\n" + " printf '%s' \"$h\"\n" + "}\n" "{ printf '{'; sep=''; " "find . -path ./graphify-out -prune -o -type f -print | " "sed 's|^\\./||' | while read -r f; do " + 'h=$(digest_of "$f"); ' 'printf \'%s"%s":{"mtime":1,"seen":2,' - '"ast_hash":"0123456789abcdef0123456789abcdef",' - "\"semantic_hash\":\"0123456789abcdef0123456789abcdef\"}' \"$sep\" \"$f\"; " + '"ast_hash":"%s",' + "\"semantic_hash\":\"%s\"}' \"$sep\" \"$f\" \"$h\" \"$h\"; " "sep=','; done; printf '}'; } > graphify-out/manifest.json\n" ) if complete From 5a6d16db9d02001e0b751a33bf0e0046f26204b1 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 09:17:59 -0700 Subject: [PATCH 15/33] Graph lifecycle: count the pin's byte-decided .m dispatch as unsupported The pinned _get_extractor returns None for a .m carrying no Objective-C directive (MATLAB/Octave), even though .m is in the suffix table, and the row is stamped anyway because that short circuit carries neither marker the failed-source rule looks for. Classify it from the same immutable pre-provider bytes every other input is classified from: no directive is unsupported, an unreadable copy stays unclassified and partial. --- docs/context-graph-lifecycle.md | 20 ++++++ src/code_mower/context_graph_lifecycle.py | 74 ++++++++++++++++++++++- tests/test_context_graph_lifecycle.py | 62 +++++++++++++++++++ 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index d44fcebd..5cdc87f0 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -443,6 +443,26 @@ dispatch is therefore counted and reported on its own line (`unsupported_inputs`) rather than folded into `indexed_files`, and zero nodes for a stamped file is a complete *read* of that file and nothing more. +### Where the boundary runs + +`classify_file` is the eligibility oracle. What it deterministically calls +not-code — every suffix outside its registry included — is not in the code +denominator, does not make a run partial, and is not counted anywhere; there is +no all-non-code skipped tally. `unsupported_inputs` counts the inputs it *does* +call code that then reach a dispatch with no extractor: the static table +difference, a code shebang with no dispatch entry, and a `.m` whose bytes carry +no Objective-C directive. That last one is decided from the bytes, not a table +— `.m` is Objective-C or MATLAB/Octave, the suffix map routes it to the +Objective-C extractor, and the pin returns no extractor for a `.m` without an +Objective-C directive rather than force-parsing MATLAB into garbage. The row is +stamped regardless, so counting it as an indexed file would report work that +did not happen. + +Everything else stays partial: an eligible code input that failed, one whose +postcondition is unknown, one whose row disagrees with the bytes it was given, +one this build could not classify against the pin's own dispatch, and a +zero-node result that cannot be told apart from a failure. + Blank rows are the cases the pin's rule makes blank: an extractor error or an anomalous zero-node extract. They stay `partial` here. The evaluation's clean-room repeat that exited zero in 1.63 seconds requeued 54 entries; the diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index b19c94a1..4042f4ba 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -2306,6 +2306,27 @@ def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: #: the files this build says were indexed. _PROVIDER_UNSUPPORTED_EXTENSIONS = frozenset({".ejs", ".ets", ".r"}) +#: ``.m`` is the one suffix whose dispatch the pin decides from the *bytes* +#: rather than the table: it is Objective-C or MATLAB/Octave, and +#: ``_get_extractor`` returns ``None`` for a ``.m`` carrying no Objective-C +#: directive (#1702) rather than force-parsing MATLAB through the ObjC grammar. +#: The static table difference above cannot see that, so without this branch a +#: MATLAB file would be counted as a file this build indexed on the strength of +#: a stamped row alone -- and the row *is* stamped, because a ``None`` extractor +#: short-circuits to ``{"nodes": [], "edges": []}`` carrying neither marker the +#: failed-source rule looks for. It is code either way, so it stays in the +#: denominator; what is decided here is only whether it contributed anything. +_PROVIDER_OBJC_AMBIGUOUS_SUFFIX = ".m" + +#: ``extract._OBJC_HEADER_MARKERS``: the Objective-C-only directives the pin +#: sniffs for, and the window it sniffs in (``_is_objc_header`` slices the first +#: 256 KiB). A marker past that window is invisible to the pin too, so reading +#: exactly that much decides the same way while staying bounded. +_PROVIDER_OBJC_MARKERS = ( + b"@interface", b"@protocol", b"@implementation", b"@import", b"#import", +) +_OBJC_PROBE_BYTES = 256 * 1024 + #: How much of an extensionless input is read to find its shebang. The pin #: reads the same 256 bytes and keeps only the first line. _SHEBANG_PROBE_BYTES = 256 @@ -2549,6 +2570,28 @@ def _shebang_interpreter(path: Path) -> tuple[str | None, bool]: return None, True +def _objc_source(path: Path) -> tuple[bool, bool]: + """``(objective_c, resolved)`` for one materialized ``.m`` input. + + ``(False, True)`` is a decision: the bytes carry no Objective-C directive, + so the pin's ``_get_extractor`` returns ``None`` for this file and it is + dispatched, stamped, and contributes nothing. ``(False, False)`` is a + refusal: the copy could not be read here, so which way the pin decided is + unknown and the caller carries the input as unclassified rather than + guessing. The pin's own sniff answers ``False`` on a read error, but that is + a statement about *its* read; this adapter failing to read the same bytes + proves nothing about what the provider was shown. + """ + try: + if path.is_symlink() or not path.is_file(): + return False, False + with path.open("rb") as stream: + head = stream.read(_OBJC_PROBE_BYTES) + except OSError: + return False, False + return any(marker in head for marker in _PROVIDER_OBJC_MARKERS), True + + def _provider_inputs( census: TrackedCensus, source_root: Path | None = None ) -> _ProviderInputs: @@ -2569,8 +2612,14 @@ def _provider_inputs( ``source_root`` is the materialized copy, read *before* the provider is launched -- the only moment those bytes are still exactly what this build - handed over. Without it no extensionless input can be classified, so every - one of them is unclassified and the run stays partial. + handed over. Without it no extensionless input and no ``.m`` can be + classified, so each of them is unclassified and the run stays partial. + + Two dispatch questions are answered from those bytes rather than from a + table: which interpreter an extensionless script names, and whether a ``.m`` + is Objective-C or MATLAB. The second decides only whether a code input had + an extractor at all, never whether it is code -- ``.m`` is in the pin's + extension table either way. """ dispatched: list[str] = [] unsupported: set[str] = set() @@ -2602,6 +2651,16 @@ def _provider_inputs( dispatched.append(path) if suffix.lower() in _PROVIDER_UNSUPPORTED_EXTENSIONS: unsupported.add(path) + elif suffix.lower() == _PROVIDER_OBJC_AMBIGUOUS_SUFFIX: + # The one dispatch the pin decides from the bytes. Read from the + # same pre-launch copy every other classification here reads, so + # the answer is about what the provider was handed. + host = _census_host_path(source_root, path) if source_root is not None else None + objective_c, resolved = _objc_source(host) if host is not None else (False, False) + if not resolved: + unclassified.append(path) + elif not objective_c: + unsupported.add(path) return _ProviderInputs(tuple(dispatched), frozenset(unsupported), tuple(unclassified)) @@ -2657,6 +2716,17 @@ def _read_completeness( ``indexed_files``, and zero nodes for a file whose row is stamped is a complete *read* of that file and nothing more. + The boundary, plainly. ``classify_file`` is the eligibility oracle: what it + deterministically calls not-code -- every suffix outside its registry + included -- is not in the denominator and does not make a run partial, so + there is no separate count of it. ``unsupported_inputs`` counts the inputs + it *does* call code that then reach a dispatch the pin has no extractor for: + the static table difference, a code shebang with no ``_SHEBANG_DISPATCH`` + entry, and a ``.m`` whose bytes carry no Objective-C directive. Anything + else is partial: an eligible code input that failed, one whose postcondition + is unknown, one whose row disagrees with the bytes, and one whose zero-node + result cannot be told apart from a failure. + Blank rows are the cases the pin's rule makes blank: an extractor error or an anomalous zero-node extract. They stay partial here. The clean-room repeat that exited zero in 1.63 s requeued 54 entries; the retained diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index e5b125f1..e4d4f87f 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -1635,6 +1635,68 @@ def test_unsupported_dispatch_is_counted_apart_from_indexed_files(self) -> None: self.assertEqual(result.indexed_files, 1) self.assertEqual(result.unsupported_inputs, 1) + def test_a_matlab_dot_m_is_dispatched_but_is_not_an_indexed_file(self) -> None: + """``.m`` is the one dispatch the pin decides from the bytes (#1702). + + The suffix table maps ``.m`` to the Objective-C extractor, but + ``_get_extractor`` returns ``None`` for a ``.m`` carrying no + Objective-C directive rather than force-parsing MATLAB through the ObjC + grammar. The row is stamped all the same -- that short circuit carries + neither marker the failed-source rule looks for -- so the static table + difference cannot see it and a stamped row alone would report work the + provider demonstrably did not do. An Objective-C file beside it is a + real extraction and still has to count. + """ + root = self.root / "objc" + (root / "src").mkdir(parents=True) + sources = { + "src/solver.m": b"function y = f(x)\n y = x + 1;\nend\n", + "src/Thing.m": b'#import "Thing.h"\n@implementation Thing\n@end\n', + } + for path, body in sources.items(): + (root / path).write_bytes(body) + census = census_of(*sources) + inputs = lifecycle._provider_inputs(census, root) + self.assertEqual(set(inputs.dispatched), set(sources)) + self.assertEqual(inputs.unsupported, frozenset({"src/solver.m"})) + self.assertEqual(inputs.unclassified, ()) + digests = { + path: hashlib.md5(body, usedforsecurity=False).hexdigest() + for path, body in sources.items() + } + result = lifecycle._read_completeness( + {path: manifest_row(digest) for path, digest in digests.items()}, + census, + digests, + inputs, + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + self.assertEqual(result.unsupported_inputs, 1) + + def test_a_dot_m_the_copy_cannot_be_read_for_stays_unclassified(self) -> None: + """Which way the pin dispatched it is unknown, so the run stays partial. + + The pin's own sniff answers "not Objective-C" when its read fails, but + that is a statement about its read. This adapter failing to read the + same bytes proves nothing, and guessing either way would either excuse a + real extraction or invent an unsupported one. + """ + census = census_of("src/solver.m") + inputs = lifecycle._provider_inputs(census, self.root / "absent") + self.assertEqual(inputs.dispatched, ("src/solver.m",)) + self.assertEqual(inputs.unsupported, frozenset()) + self.assertEqual(inputs.unclassified, ("src/solver.m",)) + digest = hashlib.md5(b"function y = f(x)\n", usedforsecurity=False).hexdigest() + result = lifecycle._read_completeness( + {"src/solver.m": manifest_row(digest)}, + census, + {"src/solver.m": digest}, + inputs, + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("could not classify 1 tracked inputs", " ".join(result.notes)) + def test_an_unsupported_extension_still_has_to_be_accounted_for(self) -> None: """Reported, not excused: an absent row for one is still partial.""" census = census_of(CODE_INPUT, "web/view.ejs") From d969d1205e8045a4589fc2768fbedf2e45ec861b Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 09:30:09 -0700 Subject: [PATCH 16/33] Context: bind the standalone fetch to the consuming revision `context_packets.main` invoked `fetch` without a revision, so the `revision="HEAD"` default resolved in the *registered* graph checkout. A graph built for commit A could then report required context available to a consumer working at commit B -- the fallback the guided route already refuses. The command now reads `consuming_revision(Path.cwd())`: the commit the checkout it was run in is at. A caller that is not a Git checkout names no revision, which a repository graph refuses rather than guessing; an organization connection never reads the value, so its sources, whose document versions have no reason to equal a code commit, are unaffected. `fetch`'s `revision` default drops to `None` so the same omission cannot reappear silently: a caller that cannot name the consuming revision gets a refusal from a repository graph instead of another checkout's `HEAD`. No existing caller relied on the default for a graph connection. Regressions drive the real entrypoint, not an argument-forwarding mock: same-revision success loaded back through the protected packet path, a distinct checkout at a resolvable commit the registered one has moved off, and a consumer that is no checkout at all -- each in its required and optional forms, each asserting no packet was minted. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 11 +++ src/code_mower/context_packets.py | 19 +++- tests/test_context_graph_connection.py | 119 +++++++++++++++++++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 96fc802c..a121be31 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -343,6 +343,17 @@ then fall out of the shared packet contract rather than out of new checks: - A **moved `HEAD`** makes the published generation stale for that revision, so authorization fails outright and nothing is delivered. +The revision both rules are evaluated against is the **consuming** one — the +commit the checkout doing the work is at — and it is never defaulted. A guided +session reads it from the directory it is preparing work in; `context fetch` +reads it from the directory it was run in. Neither falls back to resolving +`HEAD` in the checkout the connection was registered against, because that is a +different directory that moves on its own: a graph built for commit A would +otherwise answer work at commit B. A caller that is not in a Git checkout at all +names no revision, and a repository graph refuses rather than guessing one. +Organization context is unaffected: its sources carry document versions that +have no reason to equal a code commit. + Required context that is refused pauses the dependent work; optional context degrades and the session continues with ordinary repository tools. Claude, Codex and Devin receive byte-identical approved evidence, and no recipient diff --git a/src/code_mower/context_packets.py b/src/code_mower/context_packets.py index c6818d8d..0988c789 100644 --- a/src/code_mower/context_packets.py +++ b/src/code_mower/context_packets.py @@ -160,7 +160,7 @@ def _load(store, entry, policy, request, envelope, *, bound_revision=None): return packet -def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revision="HEAD"): +def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revision=None): """Retrieve once, or reauthorize and reuse; never redispatch automatically. Which provider answers is the connection's own saved state, read here under @@ -169,6 +169,12 @@ def fetch(store: ContextStore, name, spec, *, backend=None, refresh=False, revis organization connection; what differs is only where authorization and evidence come from, and neither kind can be mistaken for the other because the saved schema is checked before either path is taken. + + ``revision`` is the commit the *consuming* work is at, and it has no + default: a caller that cannot name it gets a refusal from a repository + graph rather than the registered checkout's ``HEAD``, which is a different + checkout that moves independently. An organization connection never reads + it, so the same default costs it nothing. """ spec = request_spec(spec, name) policy = spec["policy"] @@ -337,7 +343,16 @@ def main(argv=None): try: spec = request_spec(strict_json(sys.stdin.buffer.read(262_145)), args.connection) required = spec["policy"]["required"] - result = fetch(ContextStore(args.state_dir), args.connection, spec, refresh=args.refresh) + # The consuming checkout is the one this command was run from, not the + # one a connection was registered against. Leaving ``fetch`` to default + # to ``HEAD`` resolves that word in the *registered* graph checkout, so a + # graph for commit A could answer work at commit B -- the exact fallback + # the guided route already refuses. ``None`` when the caller is not a Git + # checkout at all: that refuses a repository graph here (it cannot name + # the revision its evidence would be for) and is ignored by an + # organization connection, whose sources version independently of code. + result = fetch(ContextStore(args.state_dir), args.connection, spec, refresh=args.refresh, + revision=consuming_revision(Path.cwd())) code = 0 except (ContextError, OSError, ValueError) as exc: result = {"status": "required_unavailable" if required else "optional_unavailable", diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index 732c8e49..868384dc 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -16,11 +16,16 @@ from __future__ import annotations +import io +import json import os import tempfile import unittest +from contextlib import chdir, redirect_stdout from datetime import datetime, timezone from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch from code_mower import context_delivery, context_packets, context_prepare, context_session from code_mower import context_graph_connection as connection @@ -354,6 +359,120 @@ def test_attachment_binds_the_pull_requests_head_not_the_checkouts(self) -> None self._attach(handle, consuming) +@unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") +class StandaloneGraphFetchCommandTests(unittest.TestCase): + """``code-mower context fetch`` run outside a guided session (issue #914). + + The guided route reads the consuming checkout's revision itself, from the + directory the session is preparing work in. The standalone command has no + session record to read it from, so what is proven here is that it reads that + revision from the checkout it was *run in*, rather than leaving ``fetch`` to + resolve the word ``HEAD`` in whichever checkout the connection happens to + have been registered against. Those are two directories that move + independently, and the second reading is exactly how evidence describing one + commit's code reaches work on another. + + The real entrypoint runs. Only the store root is injected -- the command + otherwise builds its own -- so the packet these tests read back came through + the same protected file, the same index, and the same authorization a + guided delivery goes through. + """ + + def setUp(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = make_repository(self.root) + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=PIN, indexer=indexer(graph_document()), root=self.private, + ) + self.store = ContextStore(self.private, vault=MemoryVault()) + connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + }) + + def run_command(self, cwd: Path, *, required: bool = True) -> tuple[int, dict]: + """Drive the real command from ``cwd``. No provider backend may load.""" + spec = { + "repository": "owner/repo", "work_item": "WORK-1", "recipient": "claude:builder", + "query": "parse_config", "source": "impact", + "policy": {**POLICY, "required": required}, + } + output = io.StringIO() + with patch("code_mower.context_packets.ContextStore", return_value=self.store), \ + patch("code_mower.context_packets._backend", + side_effect=AssertionError("a local graph must not reach the provider SDK")), \ + patch("sys.stdin", SimpleNamespace(buffer=io.BytesIO(json.dumps(spec).encode()))), \ + chdir(cwd), redirect_stdout(output): + code = context_packets.main(["--connection", "local-graph", "--request-stdin", "--json"]) + return code, json.loads(output.getvalue()) + + def _another_checkout_at_a_commit_this_one_is_not_on(self) -> Path: + """A second checkout at a commit the registered one has moved off. + + The clone is taken while the second commit is current, so that checkout + keeps it; the registered checkout is then put back on the commit its + graph was built from. The object stays reachable there, so the refusal + under test is a real comparison of two resolvable commits rather than a + name the graph's repository could not look up at all. + """ + first = lifecycle.resolve_revision(self.repository)[0] + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + git(self.repository, "add", ".") + git(self.repository, "commit", "-q", "-m", "consuming work") + second = lifecycle.resolve_revision(self.repository)[0] + consuming = self.root / "consuming" + git(self.root, "clone", "-q", str(self.repository), str(consuming)) + git(self.repository, "reset", "-q", "--hard", first) + self.assertEqual(lifecycle.resolve_revision(self.repository)[0], self.manifest.commit) + self.assertEqual(lifecycle.resolve_revision(consuming)[0], second) + self.assertNotEqual(second, self.manifest.commit) + return consuming + + def test_running_in_the_graphs_own_checkout_delivers_its_packet(self) -> None: + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + packet = context_packets.load_authorized( + self.store, "local-graph", report["packet_handle"], POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder", self.manifest.commit), + ) + self.assertEqual(packet.private_payload()["source_revision"], self.manifest.commit) + self.assertEqual(packet.private_payload()["binding"]["generation"], self.manifest.generation) + + def test_a_consumer_at_another_revision_is_refused_rather_than_answered(self) -> None: + consuming = self._another_checkout_at_a_commit_this_one_is_not_on() + code, report = self.run_command(consuming) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + # Refused at authorization, before anything is reserved: no packet of + # the wrong commit's evidence exists to be replayed later. + self.assertEqual(list(self.private.glob(".p-*.json")), []) + # And the discrimination is the consuming revision alone. The registered + # checkout still sits at the graph's commit, so a command that resolved + # ``HEAD`` there would have answered the request above as it answers + # this one. + self.assertEqual(self.run_command(self.repository)[1]["status"], "available") + + def test_an_optional_consumer_at_another_revision_degrades_instead_of_pausing(self) -> None: + consuming = self._another_checkout_at_a_commit_this_one_is_not_on() + code, report = self.run_command(consuming, required=False) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + self.assertEqual(list(self.private.glob(".p-*.json")), []) + + def test_a_consumer_that_is_not_a_checkout_is_refused_rather_than_defaulted(self) -> None: + """No revision at all is a refusal, not a fall back to the graph's.""" + elsewhere = self.root / "not-a-checkout" + elsewhere.mkdir() + code, report = self.run_command(elsewhere) + self.assertEqual((code, report["status"]), (1, "required_unavailable")) + self.assertEqual(list(self.private.glob(".p-*.json")), []) + code, report = self.run_command(elsewhere, required=False) + self.assertEqual((code, report["status"]), (0, "optional_unavailable")) + + @unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") class GraphConnectionStateTests(unittest.TestCase): """What the saved connection will and will not accept.""" From 9693a23d660af3fb4239242167caeb922b4ecad9 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 09:41:11 -0700 Subject: [PATCH 17/33] Graph queries: retain relationships between already-seen nodes run_query bounded the walk and the answer with one set of node ids, so a relationship whose far endpoint had already been visited was dropped from the result while the result still reported itself complete. Node expansion and relationship reporting are now bounded separately. ``expanded`` still admits each node to the frontier once, which is what keeps the traversal linear and terminating. ``reported`` keys on the provider's own edge record -- both endpoints, its relation, its normalized kind and its confidence -- so a distinct directed relationship is stated once even when both of its endpoints have already been seen. A two-way cycle now reports both directions; relationships among the definitions a path target selects as seeds are reported rather than deleted for having no unseen endpoint; a reconvergent walk keeps both edges into the node it reached twice; a self-loop reached from both sides of a ``symbol`` neighbourhood, and an edge the provider recorded twice, stay one relationship. The node budget, the depth limit, related-test filtering, ambiguity and the truncation and omission reporting are unchanged -- a retained relationship that does not fit the budget still sets ``truncated`` and raises ``provider_has_more`` rather than disappearing. Addresses codex:d48fae25459259ee08af (P2) on PR #982. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 13 +++ src/code_mower/context_graph_query.py | 32 +++++- tests/test_context_graph_query.py | 152 ++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 4 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index a121be31..9ffe2b99 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -199,6 +199,19 @@ cites it — `render calls load (inferred, hop 2, reached from parse_config, … rather than asserting a direct relationship between the seed and the node two hops away, which the graph does not carry. +What bounds the walk and what bounds the answer are two different things. A node +is stepped through once, which is what keeps a traversal linear and terminating. +A *relationship* is reported once per distinct provider edge record — its two +endpoints and its own wording together — including when both endpoints have +already been seen. So if `a` calls `b` and `b` calls `a`, both directions are +reported; relationships among the definitions a path target selects as seeds are +reported rather than dropped for having no unseen endpoint; a walk that +reconverges keeps both edges into the node it reached twice; and a self-loop +reached from both sides of a `symbol` neighbourhood, or an edge the provider +recorded twice, is one relationship. Nothing about the bound changes: a +relationship that does not fit the node budget still sets `truncated` and raises +`provider_has_more`, and the depth limit still applies. + ## Citations are validated against the bound commit Not against the working tree, which is the point. The generation binds one diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index b2fc5319..90f6a759 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -798,6 +798,11 @@ def run_query( the same question produce the same answer every time and a budget cut removes the *furthest* relationships rather than arbitrary ones. Reaching the budget sets ``truncated``; it never silently shortens the answer. + + Node expansion and relationship reporting are bounded separately. A node is + walked through once; a distinct directed relationship is reported once, + including when both of its endpoints have already been seen. Anything left + out is left out by the budget or the depth limit, and says so. """ if question not in QUESTIONS: raise ContextError("unsupported local graph question") @@ -820,7 +825,16 @@ def run_query( # and a name with more definitions than the seed bound allows is the same # uncertainty, only worse. ambiguous = len(seeds) > 1 or seed_overflow - seen = {node.id for node in seeds} + # Two separate identities, because they answer two separate questions. + # ``expanded`` bounds the *walk*: a node is stepped through once, which is + # what keeps the traversal linear and terminating. ``reported`` bounds the + # *answer*: a distinct directed relationship is stated once. Sharing one + # set between them silently deleted evidence -- if A calls B and B calls A, + # the second edge was suppressed because its endpoint had been walked, and + # every relationship among a path's seeds disappeared because all of its + # endpoints were seeds -- while the result still claimed to be complete. + expanded = {node.id for node in seeds} + reported: set[tuple[str, str, str, str, str]] = set() relations: list[Relation] = [] over_budget = False frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] @@ -829,18 +843,28 @@ def run_query( if level >= limit: continue for edge, other_id in _neighbours(graph, node.id, direction, kinds): - if other_id in seen: + # The provider's own record, endpoints and wording together: two + # parallel edges that say different things about the same pair are + # two relationships, a self-loop reached from both sides is one, + # and a byte-identical duplicate record is one. + identity = (edge.source, edge.target, edge.relation, edge.kind, edge.evidence) + if identity in reported: continue if len(relations) >= node_budget: over_budget = True break - seen.add(other_id) + reported.add(identity) reached = graph.nodes[other_id] # ``node``, not ``seed``: the relationship being reported is the one # this edge carries, between the node the walk expanded and the node # it just reached. The seed travels alongside as provenance. relations.append(Relation(node=reached, via=edge, origin=node, seed=seed, depth=level + 1)) - frontier.append((reached, seed, level + 1)) + # Reporting the edge never re-queues an endpoint the walk has + # already stepped through, so retaining these relationships costs + # the bound nothing: the frontier still holds each node once. + if other_id not in expanded: + expanded.add(other_id) + frontier.append((reached, seed, level + 1)) if over_budget: break if question == "related_tests": diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index d3fd0ed4..ed3a8904 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -676,6 +676,158 @@ def test_an_unsupported_question_is_refused(self) -> None: self.query(question="everything") +class RetainedRelationshipTests(unittest.TestCase): + """A relationship between two already-seen nodes is still evidence. + + Expansion and reporting were once bounded by one set of node ids, so a + graph that says two things about a pair had one of them deleted while the + result still reported itself complete. These build their own graphs because + the shapes that expose it -- a cycle, a seed set that is already connected, + a walk that reconverges -- are not in the shared fixture. + + The bound is unchanged: each node is still walked through once, and what a + budget or a depth limit removes is still reported as truncation. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def stated(self, result: query.QueryResult) -> set: + """Every reported relationship as the provider's own edge record.""" + return {(item.via.source, item.via.relation, item.via.target) for item in result.relations} + + def test_a_two_way_cycle_reports_both_directions(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha") + self.assertEqual(self.stated(result), + {("n-b", "calls", "n-a"), ("n-a", "calls", "n-b")}) + # The second direction is the edge it says it is, between its own two + # endpoints -- not the seed relabelled. + self.assertEqual([(item.origin.name, item.node.name) + for item in result.relations if item.depth == 2], + [("beta", "alpha")]) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_relationships_among_path_seeds_are_still_reported(self) -> None: + """Every endpoint is a seed, so the old reader answered with nothing.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/config.py", 20), + edges=[edge("n-a", "n-b", "calls")], + ) + result = query.run_query(graph, question="dependency", target="example_pkg/config.py") + self.assertEqual({item.id for item in result.seeds}, {"n-a", "n-b"}) + self.assertEqual(self.stated(result), {("n-a", "calls", "n-b")}) + + def test_a_reconvergent_walk_keeps_both_paths_into_one_node(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + node("n-d", "delta", "example_pkg/config.py", 30), + edges=[edge("n-a", "n-b", "calls"), edge("n-a", "n-c", "calls"), + edge("n-b", "n-d", "calls"), edge("n-c", "n-d", "calls")], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(self.stated(result), { + ("n-a", "calls", "n-b"), ("n-a", "calls", "n-c"), + ("n-b", "calls", "n-d"), ("n-c", "calls", "n-d"), + }) + self.assertEqual({(item.origin.name, item.node.name) + for item in result.relations if item.depth == 2}, + {("beta", "delta"), ("gamma", "delta")}) + self.assertFalse(result.truncated) + + def test_a_self_loop_is_reported_once_and_parallel_relations_stay_distinct(self) -> None: + """Endpoints alone are not the identity; the provider's wording is part of it.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-a", "calls"), + edge("n-a", "n-b", "calls"), + edge("n-a", "n-b", "references")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + self.assertEqual(sorted(self.stated(result)), [ + ("n-a", "calls", "n-a"), ("n-a", "calls", "n-b"), ("n-a", "references", "n-b"), + ]) + + def test_a_duplicated_edge_record_is_reported_once(self) -> None: + """Reached from both sides of a ``both`` walk, or written twice: one relationship.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-a", "n-b", "calls")], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 1) + self.assertFalse(result.truncated) + + def test_a_retained_relationship_the_budget_cuts_is_reported_as_truncation(self) -> None: + """The budget still bounds the answer, and still says what it removed.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha", node_budget=1) + self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_depth_still_bounds_a_walk_that_retains_relationships(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="impact", target="alpha", depth=1) + self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) + + +def cyclic_graph_document() -> dict: + """The shared fixture, plus the back-edge that makes the pair mutual. + + ``load`` already calls ``parse_config``; this adds the provider's own + record that ``parse_config`` also calls ``load``. + """ + return graph_document(edges=[*graph_edges(), edge("n-config", "n-load", "calls")]) + + +class RetainedRelationshipPacketTests(GraphWorkspace): + """What a recipient actually reads for a retained relationship.""" + + document = cyclic_graph_document() + + def documents(self) -> dict: + return {item["text"]: item for item in self.context().packet["documents"]} + + def test_a_retained_back_edge_states_and_cites_its_own_endpoints(self) -> None: + documents = self.documents() + [text] = [item for item in documents if "parse_config calls load" in item] + self.assertIn("hop 2", text) + self.assertEqual( + {citation["source"] for citation in documents[text]["citations"]}, + {"example_pkg/config.py#L12", "example_pkg/loader.py#L40"}, + ) + # The other direction between the same pair is still its own document, + # stated the way the provider recorded it. + self.assertTrue(any("load calls parse_config" in item for item in documents)) + + def test_the_rest_of_the_walk_is_unchanged(self) -> None: + documents = self.documents() + self.assertTrue(any("render calls load" in item and "reached from parse_config" in item + for item in documents)) + self.assertNotIn("provider_has_more", self.context().summary["omissions"]) + + class CallableLabelSeedTests(unittest.TestCase): """Bare names against the labels the pinned extractor actually writes. From a61b2767e23d5ce31f6653c9b1eaa91b6de41b86 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 12:07:27 -0700 Subject: [PATCH 18/33] Validate derived native dependencies and report missing graph endpoints Two exact-head Codex P2 findings on #982. `codex:45ebbfa33434552ac3c2` -- `_refuse_linked_library` checked ownership, size and regular-file status but never what the candidate *is*. The dependency names come out of load commands in the provider's own images, so a provider that writes its own linker input could name any operator-owned file and have it pass every metadata check. Each derived dependency is now read as a Mach-O and must declare `MH_DYLIB` or `MH_DYLIB_STUB` in its own header; a universal archive is held to that on every slice, and a file that is not a readable Mach-O at all is refused as such. Ownership, ancestry, checkout/home exclusions, resource bounds and Linux behaviour are unchanged. `codex:90851736427e7bca1693` -- `_edge` dropped an edge whenever an endpoint was absent from the filtered node map, conflating a declared non-code corpus (a stated scope) with an id the document never declared (missing evidence). The declared ids are now kept, the two cases are separated, and a relationship onto an undeclared endpoint marks its surviving endpoint. A traversal that seeds or reaches such a node reports `provider_partial` and delivers a `partial` packet; a traversal elsewhere in the graph is unaffected. Not truncation: no budget or depth limit cut it. Adds focused regressions for both, and documents both behaviours. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-lifecycle.md | 15 ++- docs/context-graph-queries.md | 20 +++ src/code_mower/context_graph_lifecycle.py | 95 +++++++++++++- src/code_mower/context_graph_query.py | 145 ++++++++++++++++++---- tests/test_context_graph_lifecycle.py | 141 ++++++++++++++++++++- tests/test_context_graph_query.py | 133 ++++++++++++++++++++ 6 files changed, 517 insertions(+), 32 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index 5cdc87f0..a82a5903 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -204,7 +204,20 @@ refusals as any other exposure — the filesystem root, the operator's home, the checkout, an ancestor of either — and is additionally refused unless it is a regular file within a size bound whose ancestry only this account or root may write, because a library the provider maps executable inside the boundary is -code. A referenced path that this host does not have installed is skipped: if it +code. It must also **say it is a shared library in its own header**: the name +came out of a load command in somebody else's image, and ownership and size say +who wrote a file and how big it is, not what it is — so without reading the +container, a provider that writes its own linker input picks which of the +operator's files this boundary exposes, and each one passes every check an +operator-owned file passes. The candidate is read as a Mach-O and its +`filetype` must be `MH_DYLIB` or the `MH_DYLIB_STUB` a stripped SDK ships in its +place. Not an executable, an object file or an `MH_BUNDLE`, which is reached +through `dlopen` rather than through the loader these commands drive. A +universal archive is held to that on **every** slice — the child is the +provider's interpreter, whose architecture is not necessarily this one, so the +slice loaded inside the boundary is not the slice a single check would pick — +and a file that is not a readable Mach-O at all is refused as such. A +referenced path that this host does not have installed is skipped: if it turns out to have been required, the loader fails the build naming the library it could not find. The derivation reads Mach-O images, so **Linux is unchanged** — an ELF runtime's libraries are already under the `/lib` and `/usr/lib` diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 9ffe2b99..38c1b758 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -234,6 +234,26 @@ at all is reported as `document_limit`. The scope rules are `context_graph`'s, applied twice: at parse time, so a node that could never be cited is not traversable either, and again at citation time. +## A dropped relationship is two different facts + +The provider's node list can be missing an edge's endpoint for two reasons, and +they are not reported the same way. + +The endpoint was **declared as a corpus this module does not query** — a +document, paper, image, rationale or concept. That is a scope stated in these +rules, the relationship is out of it, and the edge is pruned silently. No count +of those is kept: what a packet says about its own incompleteness is its +truncation and omission fields, not a tally of corpora never queried. + +The endpoint was **never declared at all**. The provider stated a relationship +and then described one of its ends nowhere, so this is evidence its own document +does not carry, not a scope this module chose. The surviving endpoint is +recorded, and a traversal that seeds or reaches that node reports +`provider_partial` and delivers a `partial` packet. It is not `truncated`: no +budget or depth limit cut it. Only the nodes the walk actually touches count — +a hole elsewhere in the repository is not a hole in this answer, and marking +every query partial for it would make the flag say nothing. + ## What the packet carries An ordinary `code_mower.contextPacket.v1` repository-kind packet — the same diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index 4042f4ba..be5cbf5d 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -1769,6 +1769,77 @@ def _macho_slice_dylib_names(stream: io.BufferedReader, offset: int) -> tuple[st return tuple(names) +#: The Mach-O file types dyld will open for an ``LC_LOAD_DYLIB``-family command: +#: ``MH_DYLIB``, and the ``MH_DYLIB_STUB`` a stripped SDK ships in its place. +#: Not ``MH_BUNDLE``, which is reached through ``dlopen`` rather than through +#: the load commands this derivation reads, and not an executable, an object +#: file, a core dump or a kernel extension -- none of which is a dependency an +#: image's load command can legitimately name. +_MACHO_DYLIB_FILETYPES = frozenset({0x6, 0x9}) + + +def _macho_slice_filetype(stream: io.BufferedReader, offset: int) -> int | None: + """The ``filetype`` field of one Mach-O image beginning at ``offset``.""" + stream.seek(offset) + magic = stream.read(4) + order_and_header = _MACHO_MAGICS.get(magic) + if order_and_header is None: + return None + order, header_size = order_and_header + header = stream.read(header_size - 4) + if len(header) != header_size - 4: + return None + # cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags[, reserved] + return int(struct.unpack(f"{order}6I", header[:24])[2]) + + +def _macho_filetypes(path: Path) -> tuple[int, ...] | None: + """Every slice's declared Mach-O ``filetype``, or ``None`` for a non-Mach-O. + + The two answers are different refusals for the caller, so they are kept + apart rather than collapsed into a falsehood. ``None`` is a file whose + container this module could not read at all -- not a Mach-O, truncated, a + universal header declaring slices it does not carry. A tuple is a file it + did read, and whose own declaration of what it is the caller then holds to + the shared-library rule. + + Every slice of a universal archive is read and every slice must parse: the + child is the provider's interpreter, whose architecture is not necessarily + this one, so a fat file admitted on the strength of a single readable slice + would be admitting whatever the other slices are. + """ + try: + with path.open("rb") as stream: + magic = stream.read(4) + if magic in _MACHO_FAT_MAGICS: + raw = stream.read(4) + if len(raw) != 4: + return None + count = struct.unpack(">I", raw)[0] + if not 1 <= count <= _MAX_MACHO_ARCHITECTURES: + return None + offsets = [] + for _ in range(count): + record = stream.read(20) + if len(record) != 20: + return None + # cputype, cpusubtype, offset, size, align + offsets.append(struct.unpack(">5I", record)[2]) + filetypes = [] + for offset in offsets: + filetype = _macho_slice_filetype(stream, offset) + if filetype is None: + return None + filetypes.append(filetype) + return tuple(filetypes) + if magic not in _MACHO_MAGICS: + return None + filetype = _macho_slice_filetype(stream, 0) + return None if filetype is None else (filetype,) + except (OSError, ValueError, struct.error): + return None + + def _scan_images(root: Path, *, budget: list[int]) -> Iterator[Path]: """Regular files under ``root`` that could be Mach-O images, within a budget. @@ -1843,8 +1914,11 @@ def _linked_runtime_libraries( exposing the manager's ``etc`` or ``var`` beside it is the operator data this boundary exists to withhold. Every added path is put through the same ownership and broad-exposure refusals as any other exposure, and a path that - is not a bounded regular file is refused rather than exposed on the strength - of an image having named it. + is not a bounded regular file -- or that does not declare itself a shared + library in its own Mach-O header -- is refused rather than exposed on the + strength of an image having named it. That last check is what keeps the + dependency *names*, which come out of somebody else's image, from choosing + which of the operator's files this boundary exposes. Linux is unchanged: an ELF runtime's libraries live under the ``/lib`` and ``/usr/lib`` directories the read-only runtime already names, and this @@ -1921,6 +1995,23 @@ def _refuse_linked_library(resolved: Path, *, repository: Path) -> None: "an account other than yours or root, so what the provider would load inside " "the sandbox is not what this host installed; no generation was published" ) + # Last, and the only check that reads the candidate's contents rather than + # its metadata. Ownership and size say who wrote a file and how big it is, + # not what it is, and the name comes out of a load command in somebody + # else's image -- so without this the provider chooses which of the + # operator's files the boundary exposes by writing its own linker input. + # The candidate must say it is a shared library in its own header. + filetypes = _macho_filetypes(resolved) + if filetypes is None: + raise ContextError( + "the local graph provider's runtime names a dependency that is not a readable " + "Mach-O image; refusing to expose it to the sandbox" + ) + if any(filetype not in _MACHO_DYLIB_FILETYPES for filetype in filetypes): + raise ContextError( + "the local graph provider's runtime names a dependency that is a Mach-O image " + "but not a shared library; refusing to expose it to the sandbox" + ) def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 90f6a759..d1211caa 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -360,6 +360,13 @@ class CodeGraph: edges: tuple[GraphEdge, ...] outgoing: Mapping[str, tuple[GraphEdge, ...]] incoming: Mapping[str, tuple[GraphEdge, ...]] + #: Code nodes that lost at least one relationship because its other endpoint + #: is an id the provider's node list never declared at all. Distinct from a + #: relationship onto a corpus this module deliberately does not query: that + #: endpoint *was* declared, and dropping it is a stated scope rather than + #: missing evidence. A traversal that touches one of these nodes is reporting + #: a neighbourhood the document itself could not state in full, and says so. + incomplete: frozenset[str] = frozenset() def seed_matches(self, target: str) -> tuple[tuple[GraphNode, ...], bool]: """The seeds a target names, and whether the seed bound dropped any. @@ -500,8 +507,24 @@ def _node_kind(path: str, label: str, node_type: Any) -> str: return "symbol" -def _node(value: Any) -> GraphNode | None: - """One pinned-export node, or ``None`` for a corpus this module does not query. +@dataclass(frozen=True) +class _ParsedNode: + """One provider node record: its declared id, and the code node it is or is not. + + The id survives the ``code``-only filter on purpose. An edge onto an id the + document *declared* as a document, paper, image, rationale or concept is a + relationship this module has stated it does not query; an edge onto an id + the document never declared at all is evidence the provider itself could + not state. Without keeping the declared ids those two are the same dangling + edge, and ``_edge`` cannot tell a scope decision from missing evidence. + """ + + id: str + node: GraphNode | None + + +def _node(value: Any) -> _ParsedNode: + """One pinned-export node: a queryable code node, or a declared exclusion. A non-``code`` node is dropped rather than refused. The provider indexes documents, papers, images, rationales and concepts into the same graph, and @@ -510,7 +533,9 @@ def _node(value: Any) -> GraphNode | None: every edge that named one becomes a dangling edge, which ``load_graph`` prunes. No count of the dropped records is kept or reported: what a packet states about its own incompleteness is the traversal's truncation and - omission fields, not a tally of corpora this module never queries. + omission fields, not a tally of corpora this module never queries. Their + *ids* are kept, and only so that ``_edge`` can tell this stated exclusion + apart from an endpoint the document never carried. """ record = _required(value, GRAPH_NODE_FIELDS, what="node") # Bounded text before membership: a vocabulary field is looked up in a set, @@ -519,12 +544,13 @@ def _node(value: Any) -> GraphNode | None: file_type = _text(record["file_type"], maximum=64) if file_type not in GRAPH_FILE_TYPES: raise ContextError("unsupported local graph node file type") + identifier = _text(record["id"], maximum=512) if file_type != CODE_FILE_TYPE: - return None + return _ParsedNode(id=identifier, node=None) path = _maybe_text(record["source_file"], maximum=1024) label = _text(record["label"], maximum=512) node = GraphNode( - id=_text(record["id"], maximum=512), + id=identifier, kind=_node_kind(path, label, record.get("type")), name=label, path=path, @@ -537,20 +563,49 @@ def _node(value: Any) -> GraphNode | None: # sourceless stub has nothing to hold to the rules and is exempt. if node.citation is not None: parse_graph_citation(node.citation) - return node + return _ParsedNode(id=identifier, node=node) + + +@dataclass(frozen=True) +class _ParsedEdge: + """One provider link: the relationship it is, or the evidence it costs. + + ``incomplete`` names the endpoints that *are* in the graph on a link whose + other end the document never declared. Those are the nodes whose reported + neighbourhood is smaller than the provider's own, so a traversal reaching + one of them has to say the answer is partial rather than call it complete. + Both fields empty is the third case, and the only silent one: a link whose + missing endpoints were all declared exclusions. + """ + + edge: GraphEdge | None + incomplete: tuple[str, ...] -def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge | None: - """One pinned-export link, or ``None`` if either endpoint is not in the graph. +def _edge( + value: Any, nodes: Mapping[str, GraphNode], excluded: frozenset[str] +) -> _ParsedEdge: + """One pinned-export link, and what dropping it costs when it is dropped. Pruned rather than refused, which is the pinned exporter's own treatment: ``export.py::prune_dangling_edges`` drops links whose endpoints are not in - the node set and reports a count. Two things reach this path in a real - export -- a link onto a corpus ``_node`` dropped above, and a link the - provider emitted onto an id its node list does not carry -- and neither is - a relationship this module can cite, since a citation needs a node with a - location. Dropping the edge is what makes the relationship absent from the - answer instead of present with one end unstated. + the node set and reports a count. But the two links that reach that path + are not the same fact and must not be reported as one: + + * A link onto a node ``_node`` dropped for its corpus. The document + declared that endpoint and this module has stated it does not query that + corpus, so the relationship is out of scope by a rule a recipient can + read. Dropped silently, as before. + * A link onto an id the node list never carries at all. The provider + emitted a relationship and then did not describe one of its ends, so this + is evidence the document is missing -- not a scope this module chose. + Reporting a complete answer over such a neighbourhood would state that + nothing was left out when something was. The surviving endpoint is named + so the traversal can raise ``provider_partial`` if it reaches it. + + A link whose *every* endpoint is undeclared names no node any traversal can + start from or reach, so there is nothing to attribute it to and nothing to + report: no query's answer can be narrowed by it. """ record = _required(value, GRAPH_EDGE_FIELDS, what="edge") # Bounded text first, for the same reason as a node's ``file_type``: the @@ -562,14 +617,25 @@ def _edge(value: Any, nodes: Mapping[str, GraphNode]) -> GraphEdge | None: relation = _text(record["relation"], maximum=128) source = _text(record["source"], maximum=512) target = _text(record["target"], maximum=512) - if source not in nodes or target not in nodes: - return None - return GraphEdge( - source=source, - target=target, - relation=relation, - kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), - evidence=GRAPH_CONFIDENCES[confidence], + endpoints = ((source, source in nodes), (target, target in nodes)) + if all(present for _, present in endpoints): + return _ParsedEdge( + edge=GraphEdge( + source=source, + target=target, + relation=relation, + kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), + evidence=GRAPH_CONFIDENCES[confidence], + ), + incomplete=(), + ) + if all(present or endpoint in excluded for endpoint, present in endpoints): + # Every absent end was a node the document declared and this module + # deliberately does not query. A stated scope, not missing evidence. + return _ParsedEdge(edge=None, incomplete=()) + return _ParsedEdge( + edge=None, + incomplete=tuple(endpoint for endpoint, present in endpoints if present), ) @@ -656,9 +722,12 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C if stamped is not None and _text(stamped, maximum=64) != commit: raise ContextError("local graph was built from a different commit than its generation") nodes: dict[str, GraphNode] = {} + excluded: set[str] = set() for value in raw_nodes: - node = _node(value) + parsed = _node(value) + node = parsed.node if node is None: + excluded.add(parsed.id) continue if node.id in nodes: # The provider's own validator does not check this, but neither of @@ -668,10 +737,18 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C # that carries two was not written by the pinned provider. raise ContextError("local graph node identifiers must be unique") nodes[node.id] = node + frozen = frozenset(excluded) + parsed_edges = [_edge(value, nodes, frozen) for value in raw_edges] edges = tuple(sorted( - (parsed for value in raw_edges if (parsed := _edge(value, nodes)) is not None), + (item.edge for item in parsed_edges if item.edge is not None), key=lambda edge: (edge.kind, edge.source, edge.target), )) + # Attributed to the surviving endpoint rather than counted: a traversal that + # never reaches one of these nodes is not answering over missing evidence + # and must not claim it is, and one that does reach it has to say so. + incomplete = frozenset( + endpoint for item in parsed_edges for endpoint in item.incomplete + ) return CodeGraph( generation=generation, commit=commit, @@ -679,6 +756,7 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C edges=edges, outgoing=_grouped(edges, by="source"), incoming=_grouped(edges, by="target"), + incomplete=incomplete, ) @@ -834,6 +912,12 @@ def run_query( # every relationship among a path's seeds disappeared because all of its # endpoints were seeds -- while the result still claimed to be complete. expanded = {node.id for node in seeds} + # Did this traversal read a neighbourhood the provider's own document could + # not state in full? Set from the nodes the walk actually touches, never + # from the graph as a whole: a dangling endpoint somewhere else in the + # repository is not a hole in *this* answer, and marking every query partial + # because of one would make the flag mean nothing. + incomplete = any(node.id in graph.incomplete for node in seeds) reported: set[tuple[str, str, str, str, str]] = set() relations: list[Relation] = [] over_budget = False @@ -855,6 +939,7 @@ def run_query( break reported.add(identity) reached = graph.nodes[other_id] + incomplete = incomplete or other_id in graph.incomplete # ``node``, not ``seed``: the relationship being reported is the one # this edge carries, between the node the walk expanded and the node # it just reached. The seed travels alongside as provenance. @@ -880,6 +965,12 @@ def run_query( omissions.append("provider_has_more") if ambiguous or any(item.via.evidence == "ambiguous" for item in relations): omissions.append("unresolved_entities") + if incomplete: + # The provider declared a relationship onto an end it never described, + # so the node this walk read has neighbours no reader of this document + # can name. Not ``truncated`` -- no budget and no depth limit cut this, + # the evidence was never in the artifact -- but still partial. + omissions.append("provider_partial") return QueryResult( question=question, target=target, generation=graph.generation, commit=graph.commit, seeds=seeds, relations=tuple(relations), truncated=truncated, ambiguous=ambiguous, @@ -1145,7 +1236,11 @@ def build_packet( *(["provider_partial"] if completeness == lifecycle.PARTIAL else []), ])) truncated = result.truncated or "document_limit" in omissions - packet_completeness = "partial" if truncated or completeness == lifecycle.PARTIAL else "complete" + # ``provider_partial`` covers both of its sources -- the generation's own + # partial build, added just above, and a traversal that read a relationship + # whose far end the document never declared. Neither is truncation, and a + # packet carrying either must not call itself complete. + packet_completeness = "partial" if truncated or "provider_partial" in omissions else "complete" expiry = min( _timestamp(connection["expires_at"]), current + timedelta(seconds=limits["max_age_seconds"]), diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index e4d4f87f..d5955d90 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -2274,13 +2274,18 @@ def test_a_readable_set_reaching_the_home_directory_is_refused(self) -> None: ) -def macho(path: Path, *dependencies: str) -> Path: +def macho(path: Path, *dependencies: str, filetype: int = 0x6) -> Path: """A real 64-bit Mach-O header whose load commands name ``dependencies``. A header, not a whole image: the derivation reads ``LC_LOAD_DYLIB`` out of the load-command block and nothing else, so a file with a real magic, a real command count and real commands exercises exactly the parse under test without needing a compiler on the machine running these tests. + + ``filetype`` defaults to ``MH_DYLIB``, which is what every image these tests + write stands in for. It is a parameter because what a candidate *declares + itself to be* is now a check, and the refusals need images that declare + something else. """ commands = b"" for name in dependencies: @@ -2290,13 +2295,27 @@ def macho(path: Path, *dependencies: str) -> Path: commands += struct.pack("<6I", 0x0C, 24 + len(raw), 24, 0, 0, 0) + raw # magic, cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags, reserved header = struct.pack( - "<8I", 0xFEEDFACF, 0x0100_000C, 0, 6, len(dependencies), len(commands), 0, 0 + "<8I", 0xFEEDFACF, 0x0100_000C, 0, filetype, len(dependencies), len(commands), 0, 0 ) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(header + commands) return path +def fat_macho(path: Path, *slices: bytes) -> Path: + """A universal archive carrying ``slices`` as real Mach-O images.""" + header = struct.pack(">2I", 0xCAFEBABE, len(slices)) + offset = len(header) + 20 * len(slices) + arches = b"" + body = b"" + for payload in slices: + arches += struct.pack(">5I", 0x0100_000C, 0, offset + len(body), len(payload), 0) + body += payload + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + arches + body) + return path + + class LinkedRuntimeLibraryTests(ProviderExposureFixture): """Shared libraries the exposed runtime links to from outside the exposure. @@ -2454,6 +2473,103 @@ def test_a_dependency_another_account_may_rewrite_is_refused(self) -> None: self.derive(prefix) self.assertIn("writable by", str(raised.exception)) + def test_a_dependency_that_is_not_a_binary_is_refused(self) -> None: + """The name comes out of somebody else's image, so the file must answer. + + Ownership and size say who wrote a file and how big it is, not what it + is. Without reading the container, a provider that writes its own linker + input picks which of the operator's files this boundary exposes -- a + shell profile, a keychain database, a notes file -- and each one passes + every metadata check an operator-owned file passes. + """ + secret = self.root / "documents" / "notes.txt" + secret.parent.mkdir(parents=True) + secret.write_text("an operator's file, owned by the operator\n") + prefix = self.root / "python" + self.library(prefix / "_x.so", str(secret)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + + def test_a_truncated_dependency_is_refused(self) -> None: + """A magic is not a header. Nothing is admitted on four bytes.""" + stub = self.root / "brew" / "lib" / "libcut.dylib" + stub.parent.mkdir(parents=True) + stub.write_bytes(b"\xcf\xfa\xed\xfe" + b"\0" * 8) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(stub)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + + def test_a_dependency_that_is_an_executable_is_refused(self) -> None: + """A Mach-O, and still not something a load command may name.""" + binary = macho(self.root / "brew" / "bin" / "tool", filetype=0x2) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(binary)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_dependency_that_is_a_bundle_is_refused(self) -> None: + """``MH_BUNDLE`` is reached through ``dlopen``, not through dyld's loader.""" + bundle = macho(self.root / "brew" / "lib" / "plugin.bundle", filetype=0x8) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(bundle)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_dylib_stub_is_admitted(self) -> None: + """``MH_DYLIB_STUB`` is what a stripped SDK ships in a library's place.""" + stub = macho(self.root / "brew" / "lib" / "libstub.dylib", filetype=0x9) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(stub)) + self.assertIn(str(stub), self.derive(prefix)) + + def test_a_universal_dependency_is_admitted_when_every_slice_is_a_library(self) -> None: + library = self.root / "brew" / "lib" / "libfat.dylib" + fat_macho( + library, + macho(self.root / "arm-slice").read_bytes(), + macho(self.root / "intel-slice").read_bytes(), + ) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(library)) + self.assertIn(str(library), self.derive(prefix)) + + def test_a_universal_dependency_with_a_non_library_slice_is_refused(self) -> None: + """One readable library slice is not a licence for whatever the rest are. + + The child is the provider's interpreter, whose architecture is not + necessarily this one, so the slice that gets loaded inside the boundary + is not the slice a check here would have picked. + """ + mixed = self.root / "brew" / "lib" / "libmixed.dylib" + fat_macho( + mixed, + macho(self.root / "good-slice").read_bytes(), + macho(self.root / "bad-slice", filetype=0x2).read_bytes(), + ) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(mixed)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a shared library", str(raised.exception)) + + def test_a_universal_dependency_with_an_unreadable_slice_is_refused(self) -> None: + """A fat header may declare a slice the file does not carry.""" + broken = self.root / "brew" / "lib" / "libbroken.dylib" + broken.parent.mkdir(parents=True) + header = struct.pack(">2I", 0xCAFEBABE, 1) + # An architecture record whose offset points past the end of the file. + broken.write_bytes(header + struct.pack(">5I", 0x0100_000C, 0, 4096, 32, 0)) + prefix = self.root / "python" + self.library(prefix / "_x.so", str(broken)) + with self.assertRaises(ContextError) as raised: + self.derive(prefix) + self.assertIn("not a readable Mach-O image", str(raised.exception)) + def test_more_libraries_than_the_bound_are_refused(self) -> None: brew = self.root / "brew" / "lib" names = [str(self.library(brew / f"lib{index}.dylib")) for index in range(4)] @@ -2567,6 +2683,16 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: if not any(lifecycle._under(resolved, root) for root in covered) ] untrusted = [str(path) for path in external if not lifecycle._trusted_library(path)] + # The same question for the other refusal a real host can legitimately + # hit: a dependency that resolves to something which does not declare + # itself a shared library. Asked here so a refusal over a host whose + # dependencies *are* all libraries still fails the test. + malformed = [ + str(path) + for path in external + if (types := lifecycle._macho_filetypes(path)) is None + or any(kind not in lifecycle._MACHO_DYLIB_FILETYPES for kind in types) + ] try: derived = lifecycle._linked_runtime_libraries( [prefix], covered=[prefix], repository=self.repository @@ -2577,9 +2703,11 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: # refused through its own ancestry, before any single dependency. # Either is a correct refusal; neither being true is not. self.assertTrue( - untrusted or not lifecycle._trusted_library(Path(os.path.realpath(prefix))), + untrusted + or malformed + or not lifecycle._trusted_library(Path(os.path.realpath(prefix))), "the boundary refused this host's runtime, but the runtime and " - f"every library _ssl links to are trusted: {refusal}", + f"every library _ssl links to are trusted shared libraries: {refusal}", ) return self.assertEqual( @@ -2587,6 +2715,11 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: [], "the boundary admitted a runtime whose libraries are writable by another account", ) + self.assertEqual( + malformed, + [], + "the boundary admitted a dependency that is not a shared library", + ) for resolved in external: self.assertIn( str(resolved), derived, f"{resolved} is outside the provider's boundary" diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index ed3a8904..b81795da 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -828,6 +828,139 @@ def test_the_rest_of_the_walk_is_unchanged(self) -> None: self.assertNotIn("provider_has_more", self.context().summary["omissions"]) +def missing_endpoint_document() -> dict: + """The shared fixture, plus a relationship onto an id it never declares. + + ``parse_config`` calls something the provider named ``n-ghost`` and then + described nowhere. The relationship is real and unreadable, which is not + the same fact as a relationship onto a corpus this module has stated it + does not query. + """ + return graph_document(edges=[*graph_edges(), edge("n-config", "n-ghost", "calls")]) + + +class MissingEndpointTests(unittest.TestCase): + """A dropped edge is two different facts, and must not be reported as one. + + The provider's node list can be missing an endpoint for two reasons. It + declared the endpoint as a document, paper, image, rationale or concept, + and this module has stated in its own contract that it does not query those + -- a scope, readable in the rules. Or it declared the endpoint nowhere at + all, which is evidence its own document does not carry. Pruning both + silently let a query over a neighbourhood the provider could not state in + full come back marked ``complete``. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def test_a_declared_exclusion_leaves_no_node_incomplete(self) -> None: + """The stated scope stays silent, exactly as before.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["edges"].append(edge("n-config", "n-doc", "references")) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + self.assertNotIn("n-doc", graph.nodes) + self.assertEqual(graph.incomplete, frozenset()) + + def test_an_undeclared_endpoint_marks_the_surviving_node(self) -> None: + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + self.assertEqual(graph.incomplete, frozenset({"n-config"})) + # Still pruned: a relationship with one end unstated is not citable. + self.assertNotIn("n-ghost", {edge_.target for edge_ in graph.edges}) + + def test_both_ends_of_an_undeclared_relationship_are_marked(self) -> None: + """Direction is not what decides it; being in the graph is.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-ghost", "n-a", "calls"), edge("n-b", "n-ghost", "calls")], + ) + self.assertEqual(graph.incomplete, frozenset({"n-a", "n-b"})) + + def test_a_relationship_with_no_surviving_endpoint_marks_nothing(self) -> None: + """No node any traversal can reach is narrowed by it, so nothing claims it.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=[edge("n-ghost", "n-other-ghost", "calls")], + ) + self.assertEqual(graph.incomplete, frozenset()) + self.assertEqual(graph.edges, ()) + + def test_a_mixed_relationship_counts_as_missing_evidence(self) -> None: + """One declared exclusion does not excuse the end that was never declared.""" + document = graph_document() + document["nodes"].append( + {**node("n-doc", "design.md", "docs/design.md", 1), "file_type": "document"} + ) + document["edges"].append(edge("n-config", "n-ghost", "calls")) + document["edges"].append(edge("n-load", "n-doc", "references")) + graph = query.load_graph(document, generation="a" * 32, commit="b" * 40) + self.assertEqual(graph.incomplete, frozenset({"n-config"})) + + def test_a_traversal_that_reaches_the_node_reports_partial(self) -> None: + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="symbol", target="parse_config") + self.assertIn("provider_partial", result.omissions) + # Not truncation: no budget and no depth limit cut this. The evidence + # was never in the artifact. + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_traversal_that_reaches_the_node_indirectly_reports_partial(self) -> None: + """Touched by the walk, not just seeded: the answer still spans that node.""" + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="dependency", target="load") + self.assertTrue(any(item.node.id == "n-config" for item in result.relations)) + self.assertIn("provider_partial", result.omissions) + + def test_a_traversal_elsewhere_in_the_graph_stays_complete(self) -> None: + """A hole somewhere else is not a hole in this answer. + + Marking every query partial because one node in the repository lost an + endpoint would make the flag say nothing about the answer carrying it. + """ + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + edges=[edge("n-a", "n-b", "calls"), edge("n-c", "n-ghost", "calls")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertNotIn("provider_partial", result.omissions) + self.assertFalse(result.truncated) + + +class MissingEndpointPacketTests(GraphWorkspace): + """What the recipient reads when the provider could not state a neighbourhood.""" + + document = missing_endpoint_document() + + def test_the_packet_is_partial_and_says_why(self) -> None: + context = self.context() + self.assertIn("provider_partial", context.packet["omissions"]) + self.assertEqual(context.packet["completeness"], "partial") + # A packet may be partial without being truncated; the delivery contract + # only forbids the other pairing. + self.assertFalse(context.packet["truncated"]) + self.assertIn("provider_partial", context.summary["omissions"]) + self.assertEqual(context.summary["completeness"], "partial") + + def test_the_relationships_it_could_state_are_still_stated(self) -> None: + """Partial is not empty: what the document did carry is still evidence.""" + texts = [item["text"] for item in self.context().packet["documents"]] + self.assertTrue(any("load calls parse_config" in text for text in texts)) + + class CallableLabelSeedTests(unittest.TestCase): """Bare names against the labels the pinned extractor actually writes. From 2e98ec69e865de40374e66a988d260e57b59b3c0 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 12:18:10 -0700 Subject: [PATCH 19/33] Validate the whole native container, not its header `_macho_filetypes` refused a dependency whose magic or `filetype` was wrong, and read everything else as if the file had told the truth about itself. A magic and one integer are the cheapest thing in a file to reproduce over arbitrary operator-owned bytes, and the name being checked comes out of an `LC_LOAD_DYLIB` in the provider's own image -- so the structure behind the claim is what has to hold up. Three specific gaps, all reachable by a provider that writes its own linker input: - `FAT_CIGAM` was decoded big-endian. Those are the same header's bytes reversed, so a count of 2 read as 33_554_432 and every slice offset was a number with no relation to the file. The container was not being decoded at all; the fields meant to bound the parse were never the fields on disk. - Fat slice offsets were read without their sizes. A slice could begin inside the architecture table describing it, run off the end of the file, or overlap another slice -- which leaves "which image is this" ambiguous, and an ambiguous container cannot answer a trust question. - The thin header's `ncmds`/`sizeofcmds` were read and never checked. A load-command region could be declared past the end of its own slice, reaching into the next slice's bytes to satisfy its header, or carry a command chain that did not walk to the size the header declared. One structural parse now decides what a container is, and both the dependency derivation and the exposure refusal read its answer instead of each deriving a weaker version. It decodes the universal header in the order its own magic declares, bounds the architecture table against the file's length, and holds every slice to beginning after that table, ending within the file, and not overlapping another. Within each admitted slice the thin header must be complete and its load-command region must fit inside *that slice*; the commands must then walk -- each `cmdsize` at least a command header, a multiple of the image's pointer width, within the region -- and consume the region exactly, since `sizeofcmds` is the size of all the commands and a chain that stops short leaves unexamined bytes where only commands belong. `FAT_MAGIC_64` has wider records and is refused as unrecognized rather than guessed at. Malformed containers are refused whole rather than read as far as they parse. Peak cost is one slice's command block, not the archive's: the validated slice records where its region is rather than holding its bytes. Existing trust, ownership, ancestor, checkout, home and size refusals run first and are unchanged, as are the resolved-file handling, the scan bounds and Linux behaviour. Adds regressions for swapped-fat containers, out-of-range and zero-length slices, slices inside the table, overlapping slices, oversized architecture tables, truncated headers and load-command regions, a region declared past its slice, and command chains that are miscounted, misaligned, oversized or short -- alongside valid thin and fat dylibs in both byte orders. --- docs/context-graph-lifecycle.md | 23 +- src/code_mower/context_graph_lifecycle.py | 350 +++++++++++++++------- tests/test_context_graph_lifecycle.py | 264 ++++++++++++++++ 3 files changed, 529 insertions(+), 108 deletions(-) diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md index a82a5903..0fee5551 100644 --- a/docs/context-graph-lifecycle.md +++ b/docs/context-graph-lifecycle.md @@ -216,7 +216,28 @@ through `dlopen` rather than through the loader these commands drive. A universal archive is held to that on **every** slice — the child is the provider's interpreter, whose architecture is not necessarily this one, so the slice loaded inside the boundary is not the slice a single check would pick — -and a file that is not a readable Mach-O at all is refused as such. A +and a file that is not a readable Mach-O at all is refused as such. + +A magic and a `filetype` field are not that reading. Those four bytes and that +one integer are the cheapest thing in the file to reproduce over arbitrary +operator-owned bytes, so the **structure behind them** is what is checked. The +universal header is decoded in the byte order its own magic declares — +`FAT_MAGIC` big-endian, `FAT_CIGAM` little — because reading the swapped +spelling as big-endian turns a count of two into 33 million and every slice +offset into a number with no relation to the file. The architecture table is +bounded and must fit in the file; each slice must begin after the table +describing it, end within the file, and not overlap another slice, because +overlapping slices make "which image is this" ambiguous. Within each admitted +slice the thin header must be complete and its declared load-command region +must fit inside **that slice** rather than merely inside the file, so one slice +cannot reach into the next one's bytes to satisfy its header. The commands then +have to walk: each `cmdsize` at least a command header, a multiple of the +image's pointer width, and within the region — and the chain must consume the +region exactly, since `sizeofcmds` is the size of *all* the commands and a +chain that stops short leaves unexamined bytes where only commands belong. A +64-bit universal header (`FAT_MAGIC_64`) has wider records and is refused as +unrecognized rather than guessed at. Anything malformed is refused whole rather +than read as far as it parses. A referenced path that this host does not have installed is skipped: if it turns out to have been required, the loader fails the build naming the library it could not find. The derivation reads Mach-O images, so **Linux is unchanged** diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index be5cbf5d..33cc6b9c 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -1637,10 +1637,36 @@ def _refuse_broad_readable(readable: Iterable[str], *, repository: Path) -> None b"\xfe\xed\xfa\xce": (">", 28), # 32-bit, big endian } -#: A universal ("fat") archive: a big-endian count of architecture records, each -#: naming the offset of a real Mach-O image inside the same file. A python.org +#: The 64-bit thin header's width, which is also what says an image's pointers +#: -- and so the multiple its load-command sizes must be -- are eight bytes. +_MACHO_HEADER_BYTES_64 = 32 + +#: A load command's own header: the command and its size. +_MACHO_COMMAND_HEADER_BYTES = 8 + +#: A universal ("fat") archive: a count of architecture records, each naming the +#: offset and size of a real Mach-O image inside the same file. A python.org #: interpreter ships these; a Homebrew one does not. -_MACHO_FAT_MAGICS = frozenset({b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca"}) +#: +#: The two spellings are the same header written in the two byte orders, and the +#: magic is what says which: ``FAT_MAGIC`` reads as ``0xCAFEBABE`` big-endian, so +#: the count and the architecture records that follow it are big-endian too; +#: ``FAT_CIGAM`` is those four bytes reversed, and its fields are reversed with +#: them. Reading a ``FAT_CIGAM`` header as big-endian -- which is what this did +#: -- turns a count of 2 into 33_554_432 and a slice offset into a number with +#: no relation to the file, so the container was not being decoded at all. +_MACHO_FAT_ORDERS: Mapping[bytes, str] = { + b"\xca\xfe\xba\xbe": ">", # FAT_MAGIC + b"\xbe\xba\xfe\xca": "<", # FAT_CIGAM +} + +#: One architecture record: cputype, cpusubtype, offset, size, align. +_MACHO_FAT_ARCH_BYTES = 20 + +#: The 64-bit universal header (``FAT_MAGIC_64``) is deliberately absent. Its +#: records are a different width with 64-bit offsets, so admitting it would mean +#: a second parse; an unrecognized container is refused rather than guessed at, +#: and no runtime this boundary has had to derive ships one. #: The load commands that name a library the image will make dyld find. Weak, #: re-exported and upward links are included: a weak dependency that *is* @@ -1684,89 +1710,245 @@ def _refuse_broad_readable(readable: Iterable[str], *, repository: Path) -> None _NOT_MACHO_DIRECTORIES = frozenset({"__pycache__", ".git", "include", "man", "doc", "docs"}) -def _macho_dylib_names(path: Path) -> tuple[str, ...]: - """Every library path a Mach-O image at ``path`` asks dyld to load. +@dataclass(frozen=True) +class _MachoSlice: + """One structurally validated Mach-O image inside a file. + + Produced only by :func:`_macho_slices`, and only after that function has + established that every field below names a region the file actually carries. + Holding the located load-command region rather than its bytes keeps the peak + cost of reading a universal archive one slice's block rather than all of + them at once. + """ - Read out of the image's own load commands rather than out of ``otool``: - deriving the boundary must not itself depend on a developer tool being - installed, and a parse that reads a bounded header is a smaller thing to - trust than a subprocess. A file that is not a Mach-O -- which is almost - everything under an install prefix -- costs four bytes and returns nothing. + order: str + """The byte order this image declared in its own magic.""" + + filetype: int + """What the image says it is: ``MH_DYLIB``, ``MH_EXECUTE``, and so on.""" + + ncmds: int + """How many load commands the region carries. Validated to walk exactly.""" + + commands_offset: int + """Absolute offset in the file of the first load command.""" + + sizeofcmds: int + """Size of the whole load-command region, bounded and known to be present.""" + + +def _macho_slices(path: Path) -> tuple[_MachoSlice, ...] | None: + """Every Mach-O image in ``path``, or ``None`` if it is not a Mach-O file. + + The one structural parse: what a container *is* is decided here, and both + the dependency derivation and the exposure refusal read its answer rather + than each re-deriving a weaker version of it. ``None`` means this module + could not read the file as a Mach-O at all, which covers a file that is not + one, a truncated one, and a malformed one -- a universal header declaring + slices the file does not carry, slices that overlap each other or the + architecture table, a load-command region that runs past its slice, or a + command chain that does not walk to exactly the size the header declared. + + Nothing here is inferred from a name, a size or a suffix. Every bound is + checked against the file's own length, taken from the open descriptor so + that what is measured is what is read. The whole parse is bounded before it + allocates: at most :data:`_MAX_MACHO_ARCHITECTURES` slices, each with at + most :data:`_MAX_MACHO_COMMAND_BYTES` of load commands, read one at a time. + + A header alone is not enough to make this judgement, which is why it is made + here rather than at a magic and a ``filetype`` field. Those four bytes and + that one integer are the cheapest thing in the file for a provider that + writes its own linker input to reproduce over arbitrary operator-owned + bytes; the structure behind them is not. """ try: with path.open("rb") as stream: + size = os.fstat(stream.fileno()).st_size magic = stream.read(4) - if magic in _MACHO_FAT_MAGICS: - return _macho_fat_dylib_names(stream) + order = _MACHO_FAT_ORDERS.get(magic) + if order is not None: + return _macho_fat_slices(stream, order=order, size=size) if magic not in _MACHO_MAGICS: - return () - stream.seek(0) - return _macho_slice_dylib_names(stream, 0) + return None + located = _macho_slice(stream, offset=0, limit=size) + return None if located is None else (located,) except (OSError, ValueError, struct.error): - # An unreadable or truncated image says nothing about what the runtime - # needs. The build still fails if it was a library the provider loads, - # and it fails as dyld naming the image rather than as this module - # guessing at one. - return () + return None -def _macho_fat_dylib_names(stream: io.BufferedReader) -> tuple[str, ...]: - """The union over a universal archive's slices. +def _macho_fat_slices( + stream: io.BufferedReader, *, order: str, size: int +) -> tuple[_MachoSlice, ...] | None: + """The validated slices of a universal archive, or ``None`` if malformed. - The union rather than the slice matching this process: the child is the - provider's interpreter, whose architecture is not necessarily this one, and - every slice's dependencies are paths on the same host. + Every slice must parse, not merely one: the child is the provider's + interpreter, whose architecture is not necessarily this one, so a fat file + admitted on the strength of a readable first slice would be admitting + whatever the rest of it holds. """ - count = struct.unpack(">I", stream.read(4))[0] - if count > _MAX_MACHO_ARCHITECTURES: - return () - offsets = [] + raw = stream.read(4) + if len(raw) != 4: + return None + count = struct.unpack(f"{order}I", raw)[0] + if not 1 <= count <= _MAX_MACHO_ARCHITECTURES: + return None + table_end = 8 + _MACHO_FAT_ARCH_BYTES * count + if table_end > size: + # The header promises more architecture records than the file carries. + return None + extents: list[tuple[int, int]] = [] for _ in range(count): - record = stream.read(20) - if len(record) != 20: - return () + record = stream.read(_MACHO_FAT_ARCH_BYTES) + if len(record) != _MACHO_FAT_ARCH_BYTES: + return None # cputype, cpusubtype, offset, size, align - offsets.append(struct.unpack(">5I", record)[2]) - names: dict[str, None] = {} - for offset in offsets: - for name in _macho_slice_dylib_names(stream, offset): - names.setdefault(name, None) - return tuple(names) + _, _, offset, length, _ = struct.unpack(f"{order}5I", record) + if length <= 0 or offset < table_end or offset + length > size: + # A slice that starts inside the header this is reading, or that + # runs off the end of the file, describes a file other than this + # one. Both fields came out of the file, so both are bounded + # 32-bit values and the sum cannot overflow a Python int. + return None + extents.append((offset, length)) + ordered = sorted(extents) + for (offset, length), (next_offset, _) in zip(ordered, ordered[1:]): + if offset + length > next_offset: + # Overlapping slices make "which image is this" ambiguous, and an + # ambiguous container is not one to answer a trust question from. + return None + slices = [] + for offset, length in extents: + located = _macho_slice(stream, offset=offset, limit=length) + if located is None: + return None + slices.append(located) + return tuple(slices) + +def _macho_slice(stream: io.BufferedReader, *, offset: int, limit: int) -> _MachoSlice | None: + """One Mach-O image of ``limit`` bytes at ``offset``, fully validated. -def _macho_slice_dylib_names(stream: io.BufferedReader, offset: int) -> tuple[str, ...]: - """The ``LC_LOAD_DYLIB`` family of one Mach-O image beginning at ``offset``.""" + ``limit`` is the slice's own extent -- the whole file for a thin image, the + architecture record's declared size for a fat one -- and every read is held + inside it. A slice may not reach past itself into another slice's bytes to + satisfy its header. + """ stream.seek(offset) magic = stream.read(4) order_and_header = _MACHO_MAGICS.get(magic) if order_and_header is None: - return () + return None order, header_size = order_and_header + if limit < header_size: + return None header = stream.read(header_size - 4) if len(header) != header_size - 4: - return () + return None # cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags[, reserved] - ncmds, sizeofcmds = struct.unpack(f"{order}6I", header[:24])[3:5] + _, _, filetype, ncmds, sizeofcmds, _ = struct.unpack(f"{order}6I", header[:24]) if sizeofcmds > _MAX_MACHO_COMMAND_BYTES: - return () + return None + if header_size + sizeofcmds > limit: + # The declared load-command region does not fit in the image that + # declared it. + return None + if ncmds * _MACHO_COMMAND_HEADER_BYTES > sizeofcmds: + # Every load command carries at least its own command and size, so a + # region this small cannot hold the count claimed over it. + return None block = stream.read(sizeofcmds) + if len(block) != sizeofcmds: + return None + pointer = 8 if header_size == _MACHO_HEADER_BYTES_64 else 4 + if not _macho_commands_walk(block, order=order, ncmds=ncmds, pointer=pointer): + return None + return _MachoSlice( + order=order, + filetype=int(filetype), + ncmds=int(ncmds), + commands_offset=offset + header_size, + sizeofcmds=int(sizeofcmds), + ) + + +def _macho_commands_walk(block: bytes, *, order: str, ncmds: int, pointer: int) -> bool: + """Whether ``block`` is exactly ``ncmds`` well-formed load commands. + + Exactly: the chain must consume the region the header declared, with nothing + left over and nothing missing. ``sizeofcmds`` is defined as the total size + of all the load commands, so a chain that stops short leaves bytes in a + region that is supposed to be nothing but commands, and a chain that would + run past has already lied about one command's size. Either way the image is + not describing itself, and this refuses rather than reading as far as it can + and treating the prefix as the truth. + + ``pointer`` is the image's pointer width, which is the multiple dyld + requires each ``cmdsize`` to be: 8 for a 64-bit image, 4 for a 32-bit one. + """ + position = 0 + for _ in range(ncmds): + if position + _MACHO_COMMAND_HEADER_BYTES > len(block): + return False + _, size = struct.unpack_from(f"{order}2I", block, position) + if size < _MACHO_COMMAND_HEADER_BYTES or size % pointer: + return False + if position + size > len(block): + return False + position += size + return position == len(block) + + +def _macho_dylib_names(path: Path) -> tuple[str, ...]: + """Every library path a Mach-O image at ``path`` asks dyld to load. + + Read out of the image's own load commands rather than out of ``otool``: + deriving the boundary must not itself depend on a developer tool being + installed, and a parse that reads a bounded header is a smaller thing to + trust than a subprocess. A file that is not a Mach-O -- which is almost + everything under an install prefix -- costs four bytes and returns nothing. + + The union over a universal archive's slices rather than the slice matching + this process: the child is the provider's interpreter, whose architecture is + not necessarily this one, and every slice's dependencies are paths on the + same host. + + A file this cannot read structurally contributes nothing. An unreadable or + malformed image says nothing about what the runtime needs; the build still + fails if it was a library the provider loads, and it fails as dyld naming + the image rather than as this module guessing at one. + """ + slices = _macho_slices(path) + if not slices: + return () names: dict[str, None] = {} + try: + with path.open("rb") as stream: + for located in slices: + stream.seek(located.commands_offset) + block = stream.read(located.sizeofcmds) + if len(block) != located.sizeofcmds: + return () + _collect_dylib_names(block, order=located.order, ncmds=located.ncmds, into=names) + except (OSError, ValueError, struct.error): + return () + return tuple(names) + + +def _collect_dylib_names( + block: bytes, *, order: str, ncmds: int, into: dict[str, None] +) -> None: + """The ``LC_LOAD_DYLIB`` family in an already-validated command region.""" position = 0 for _ in range(ncmds): - if position + 8 > len(block): - break command, size = struct.unpack_from(f"{order}2I", block, position) - if size < 8 or position + size > len(block): - break if command in _MACHO_DYLIB_COMMANDS and size >= 24: name_offset = struct.unpack_from(f"{order}I", block, position + 8)[0] if 8 <= name_offset < size: raw = block[position + name_offset : position + size] name = raw.split(b"\0", 1)[0].decode("utf-8", "replace") if name: - names.setdefault(name, None) + into.setdefault(name, None) position += size - return tuple(names) #: The Mach-O file types dyld will open for an ``LC_LOAD_DYLIB``-family command: @@ -1778,66 +1960,18 @@ def _macho_slice_dylib_names(stream: io.BufferedReader, offset: int) -> tuple[st _MACHO_DYLIB_FILETYPES = frozenset({0x6, 0x9}) -def _macho_slice_filetype(stream: io.BufferedReader, offset: int) -> int | None: - """The ``filetype`` field of one Mach-O image beginning at ``offset``.""" - stream.seek(offset) - magic = stream.read(4) - order_and_header = _MACHO_MAGICS.get(magic) - if order_and_header is None: - return None - order, header_size = order_and_header - header = stream.read(header_size - 4) - if len(header) != header_size - 4: - return None - # cputype, cpusubtype, filetype, ncmds, sizeofcmds, flags[, reserved] - return int(struct.unpack(f"{order}6I", header[:24])[2]) - - def _macho_filetypes(path: Path) -> tuple[int, ...] | None: """Every slice's declared Mach-O ``filetype``, or ``None`` for a non-Mach-O. The two answers are different refusals for the caller, so they are kept apart rather than collapsed into a falsehood. ``None`` is a file whose - container this module could not read at all -- not a Mach-O, truncated, a - universal header declaring slices it does not carry. A tuple is a file it - did read, and whose own declaration of what it is the caller then holds to - the shared-library rule. - - Every slice of a universal archive is read and every slice must parse: the - child is the provider's interpreter, whose architecture is not necessarily - this one, so a fat file admitted on the strength of a single readable slice - would be admitting whatever the other slices are. + container this module could not read as a Mach-O -- not one at all, + truncated, or structurally malformed. A tuple is a file whose structure it + did validate, and whose own declaration of what it is the caller then holds + to the shared-library rule. """ - try: - with path.open("rb") as stream: - magic = stream.read(4) - if magic in _MACHO_FAT_MAGICS: - raw = stream.read(4) - if len(raw) != 4: - return None - count = struct.unpack(">I", raw)[0] - if not 1 <= count <= _MAX_MACHO_ARCHITECTURES: - return None - offsets = [] - for _ in range(count): - record = stream.read(20) - if len(record) != 20: - return None - # cputype, cpusubtype, offset, size, align - offsets.append(struct.unpack(">5I", record)[2]) - filetypes = [] - for offset in offsets: - filetype = _macho_slice_filetype(stream, offset) - if filetype is None: - return None - filetypes.append(filetype) - return tuple(filetypes) - if magic not in _MACHO_MAGICS: - return None - filetype = _macho_slice_filetype(stream, 0) - return None if filetype is None else (filetype,) - except (OSError, ValueError, struct.error): - return None + slices = _macho_slices(path) + return None if slices is None else tuple(located.filetype for located in slices) def _scan_images(root: Path, *, budget: list[int]) -> Iterator[Path]: @@ -1914,11 +2048,13 @@ def _linked_runtime_libraries( exposing the manager's ``etc`` or ``var`` beside it is the operator data this boundary exists to withhold. Every added path is put through the same ownership and broad-exposure refusals as any other exposure, and a path that - is not a bounded regular file -- or that does not declare itself a shared - library in its own Mach-O header -- is refused rather than exposed on the + is not a bounded regular file -- or that is not a structurally valid Mach-O + declaring itself a shared library -- is refused rather than exposed on the strength of an image having named it. That last check is what keeps the dependency *names*, which come out of somebody else's image, from choosing - which of the operator's files this boundary exposes. + which of the operator's files this boundary exposes, and it is the whole + container rather than a magic and a ``filetype`` field that has to hold up: + see :func:`_macho_slices`. Linux is unchanged: an ELF runtime's libraries live under the ``/lib`` and ``/usr/lib`` directories the read-only runtime already names, and this diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index d5955d90..fbaf1cb8 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -2726,6 +2726,270 @@ def test_the_real_ssl_extension_dependencies_are_covered(self) -> None: ) +def fat_container( + path: Path, + *slices: bytes, + order: str = ">", + magic: int = 0xCAFEBABE, + count: int | None = None, + extents: list[tuple[int, int]] | None = None, +) -> Path: + """A universal archive with every field of its header under the test's hand. + + :func:`fat_macho` writes a well-formed one. This writes whatever a malformed + or hostile one would say: a count that disagrees with the records, an offset + inside the header being read, a slice running off the end of the file, two + slices claiming the same bytes, or the whole header in the other byte order. + ``extents`` replaces the computed ``(offset, size)`` of each record; the + payloads are still laid out end to end after the table, so a record can + describe a region the file does or does not carry. + """ + declared = len(slices) if count is None else count + header = struct.pack(f"{order}2I", magic, declared) + start = len(header) + 20 * len(slices) + body = b"" + placed = [] + for payload in slices: + placed.append((start + len(body), len(payload))) + body += payload + records = extents if extents is not None else placed + arches = b"" + for offset, size in records: + arches += struct.pack(f"{order}5I", 0x0100_000C, 0, offset, size, 0) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(header + arches + body) + return path + + +class MachoContainerStructureTests(unittest.TestCase): + """What a candidate's own container has to establish before it is exposed. + + Ownership, size and regular-file status say who wrote a file and how big it + is. A magic and a ``filetype`` field say what four bytes and one integer + claim -- and the name being checked came out of an ``LC_LOAD_DYLIB`` in the + provider's own image, so those are the cheapest thing in the file for a + provider that writes its own linker input to reproduce over arbitrary + operator-owned bytes. + + These hold the parse to the structure behind the claim: the container is + decoded in the byte order its magic declares, every declared region is + checked against the length of the file that carries it, slices may not + overlap each other or the table describing them, and each image's + load-command region must be present and walk to exactly the size its header + declared. A container that fails any of it is refused rather than read as + far as it parses. + """ + + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.addCleanup(self._directory.cleanup) + self.root = Path(self._directory.name).resolve() + + def thin(self, *dependencies: str, filetype: int = 0x6) -> bytes: + return macho(self.root / "payload", *dependencies, filetype=filetype).read_bytes() + + # -- containers that are read ------------------------------------------ + + def test_a_thin_library_reads_as_its_filetype_and_dependencies(self) -> None: + library = macho(self.root / "libthin.dylib", "/usr/lib/libz.dylib") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6,)) + self.assertEqual(lifecycle._macho_dylib_names(library), ("/usr/lib/libz.dylib",)) + + def test_a_universal_library_reads_as_every_slice(self) -> None: + library = fat_container(self.root / "libfat.dylib", self.thin(), self.thin(filetype=0x9)) + self.assertEqual(lifecycle._macho_filetypes(library), (0x6, 0x9)) + + def test_a_swapped_universal_header_is_decoded_in_its_own_order(self) -> None: + """``FAT_CIGAM`` is the same header written the other way round. + + Read big-endian -- which is what this did -- a count of two reads as + 33_554_432 and every slice offset is a number with no relation to the + file, so a legitimate archive was refused as malformed and the fields + that were supposed to bound the parse were never the fields on disk. + """ + library = fat_container( + self.root / "libswapped.dylib", + self.thin(), + self.thin(filetype=0x9), + order="<", + ) + # The same header value, written the other way: ``FAT_CIGAM`` on disk. + self.assertEqual(library.read_bytes()[:4], b"\xbe\xba\xfe\xca") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6, 0x9)) + + def test_a_swapped_universal_header_carries_its_dependencies(self) -> None: + """The same decode, through the derivation that reads load commands.""" + library = fat_container( + self.root / "libswapped.dylib", + self.thin("/usr/lib/libz.dylib"), + order="<", + ) + self.assertEqual(lifecycle._macho_dylib_names(library), ("/usr/lib/libz.dylib",)) + + def test_an_image_with_no_load_commands_is_read(self) -> None: + """Zero commands and a zero-length region agree with each other.""" + library = macho(self.root / "libbare.dylib") + self.assertEqual(lifecycle._macho_filetypes(library), (0x6,)) + + # -- containers that are refused --------------------------------------- + + def refused(self, path: Path) -> None: + self.assertIsNone(lifecycle._macho_filetypes(path)) + self.assertEqual(lifecycle._macho_dylib_names(path), ()) + + def test_a_file_that_is_not_a_macho_is_refused(self) -> None: + other = self.root / "notes.txt" + other.write_bytes(b"not a mach-o at all") + self.refused(other) + + def test_a_sixty_four_bit_universal_header_is_refused_as_unrecognized(self) -> None: + """``FAT_MAGIC_64`` has wider records; guessing at them is not reading them.""" + container = self.root / "lib64fat.dylib" + container.write_bytes(struct.pack(">2I", 0xCAFEBABF, 1) + b"\0" * 32) + self.refused(container) + + def test_a_slice_running_past_the_end_of_the_file_is_refused(self) -> None: + payload = self.thin() + container = fat_container( + self.root / "libpast.dylib", payload, extents=[(28, len(payload) + 4096)] + ) + self.refused(container) + + def test_a_slice_beginning_past_the_end_of_the_file_is_refused(self) -> None: + container = fat_container( + self.root / "libgone.dylib", self.thin(), extents=[(0x10_0000, 32)] + ) + self.refused(container) + + def test_a_slice_inside_the_architecture_table_is_refused(self) -> None: + """A slice may not start in the header that is describing it.""" + payload = self.thin() + container = fat_container( + self.root / "liboverlap.dylib", payload, extents=[(4, len(payload))] + ) + self.refused(container) + + def test_a_zero_length_slice_is_refused(self) -> None: + container = fat_container(self.root / "libempty.dylib", self.thin(), extents=[(28, 0)]) + self.refused(container) + + def test_overlapping_slices_are_refused(self) -> None: + """Two records claiming the same bytes make "which image is this" ambiguous.""" + payload = self.thin() + container = fat_container( + self.root / "libambiguous.dylib", + payload, + payload, + extents=[(48, len(payload)), (48 + len(payload) // 2, len(payload))], + ) + self.refused(container) + + def test_an_architecture_table_larger_than_the_file_is_refused(self) -> None: + """A count is a promise about bytes the file has to carry.""" + container = fat_container(self.root / "libclaims.dylib", self.thin(), count=8) + self.refused(container) + + def test_more_architectures_than_the_bound_are_refused(self) -> None: + container = self.root / "libmany.dylib" + count = lifecycle._MAX_MACHO_ARCHITECTURES + 1 + container.write_bytes(struct.pack(">2I", 0xCAFEBABE, count) + b"\0" * (20 * count)) + self.refused(container) + + def test_a_zero_architecture_universal_header_is_refused(self) -> None: + container = self.root / "libnone.dylib" + container.write_bytes(struct.pack(">2I", 0xCAFEBABE, 0)) + self.refused(container) + + def test_a_truncated_thin_header_is_refused(self) -> None: + container = self.root / "libcut.dylib" + container.write_bytes(b"\xcf\xfa\xed\xfe" + b"\0" * 8) + self.refused(container) + + def test_a_truncated_load_command_region_is_refused(self) -> None: + """The header declares a region; the file has to carry all of it.""" + whole = self.thin("/usr/lib/libz.dylib") + container = self.root / "libshort.dylib" + container.write_bytes(whole[:-8]) + self.refused(container) + + def test_a_load_command_region_declared_past_the_slice_is_refused(self) -> None: + """A slice may not reach into the next slice's bytes to satisfy its header. + + The bytes that follow are a real, complete image, so a parse bounded by + the file rather than by the slice would read them and admit this. + """ + payload = self.thin() + stretched = bytearray(payload) + struct.pack_into(" None: + """Every command carries at least its own command and size.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + """``sizeofcmds`` is the size of *all* the commands, not of a prefix. + + A chain that leaves bytes over means the region holds something other + than the commands it was declared to hold, and reading the prefix as if + it were the whole truth is how a trailing record goes unexamined. + """ + image = bytearray(self.thin("/usr/lib/libz.dylib", "/usr/lib/libiconv.dylib")) + struct.pack_into(" None: + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + """dyld requires each ``cmdsize`` to be a multiple of the pointer width.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + declared = struct.unpack_from(" None: + """A command of no size is an unbounded walk, not a record.""" + image = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" None: + image = bytearray(self.thin()) + struct.pack_into(" None: + """One readable slice is not a licence for whatever the rest are.""" + broken = bytearray(self.thin("/usr/lib/libz.dylib")) + struct.pack_into(" Date: Mon, 14 Sep 2026 12:21:17 -0700 Subject: [PATCH 20/33] Declare the adjacent-slice zip as intentionally unequal in length Ruff B905 flags the adjacent-pair loop over the sorted fat-slice extents: zip(ordered, ordered[1:]) pairs sequences that intentionally differ in length by one, so the shorter tail is the terminating condition rather than a bug. Declare that explicitly with strict=False; the overlap bounds logic is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_graph_lifecycle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index 33cc6b9c..b6850a08 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -1811,7 +1811,7 @@ def _macho_fat_slices( return None extents.append((offset, length)) ordered = sorted(extents) - for (offset, length), (next_offset, _) in zip(ordered, ordered[1:]): + for (offset, length), (next_offset, _) in zip(ordered, ordered[1:], strict=False): if offset + length > next_offset: # Overlapping slices make "which image is this" ambiguous, and an # ambiguous container is not one to answer a trust question from. From 05a3cc2807330fd12f119dd908563d55bc71c478 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 13:49:15 -0700 Subject: [PATCH 21/33] Bound graph packets to the selected document budget build_packet normalized the policy and then built documents to this adapter's own MAX_DOCUMENTS alone. The shared contract defaults max_documents to 5 and enforces it when the packet is loaded, so a wide answer -- six citable relationships under the default policy -- was built complete at six and then refused on the way into the protected store: packet_invalid, required work paused over evidence the graph actually had. The budget now reaches the traversal. _documents takes the selected allowance and builds to the smaller of it and MAX_DOCUMENTS, so a policy may only tighten what one packet carries and never lift the adapter's ceiling. A relationship the budget leaves out already sets the existing dropped path: document_limit, truncated, and partial completeness, in the packet and in the metadata summary alike. Traversal order, citation validation, revision and generation binding are untouched. Regressions cover the real protected-store paths: the standalone fetch command under the default policy in both required and optional modes, and the guided prepare -> authorized load -> deliver -> reuse path, each asserting five documents, explicit truncation, document_limit, and no packet_invalid. Explicit lower budgets, a budget above the adapter ceiling that does not lift it, and the exact-boundary case that must not claim truncation are covered at the packet builder. Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_graph_query.py | 18 +++- tests/test_context_graph_connection.py | 119 ++++++++++++++++++++++++- tests/test_context_graph_query.py | 101 +++++++++++++++++++++ 3 files changed, 234 insertions(+), 4 deletions(-) diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index d1211caa..905e461f 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -235,6 +235,10 @@ #: delivery. DEFAULT_NODE_BUDGET = 40 MAX_NODE_BUDGET = 200 +#: The adapter's own hard ceiling on documents per packet. The selected +#: policy's ``max_documents`` applies on top of it and only ever downward: a +#: packet carries the smaller of the two, and a policy asking for more than this +#: still gets this. MAX_DOCUMENTS = 16 MAX_CITATIONS_PER_DOCUMENT = 10 MAX_SEEDS = 8 @@ -1108,7 +1112,7 @@ class PacketDraft: def _documents( - result: QueryResult, validator: CitationValidator + result: QueryResult, validator: CitationValidator, budget: int ) -> tuple[list[dict[str, Any]], list[str]]: """One document per relationship, with only citations that actually resolve. @@ -1116,14 +1120,22 @@ def _documents( commit is dropped rather than downgraded: the packet's whole claim is that its citations point at the immutable tree, and evidence that cannot be pointed at is not weaker evidence, it is none. + + ``budget`` is the selected policy's document allowance. The effective + ceiling is the smaller of it and this adapter's own ``MAX_DOCUMENTS``, so a + policy may only tighten what one packet carries, never lift the adapter + limit. Answering past the policy's budget is not an option: the contract + refuses such a packet at delivery, which turns an honestly truncated answer + into no answer at all. """ omissions: list[str] = [] documents: list[dict[str, Any]] = [] + limit = min(MAX_DOCUMENTS, budget) dropped = False unvalidated = False citations_used = 0 for item in result.relations: - if len(documents) >= MAX_DOCUMENTS: + if len(documents) >= limit: dropped = True break # Both endpoints of the edge the sentence states, never the seed the @@ -1224,7 +1236,7 @@ def build_packet( raise ContextError("unsupported local graph completeness") if not result.resolved: raise ContextError("local graph query resolved no symbol or path to cite") - documents, dropped = _documents(result, validator) + documents, dropped = _documents(result, validator, limits["max_documents"]) if not documents: raise ContextError("local graph query produced no citable evidence") omissions = list(dict.fromkeys([ diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index 868384dc..8f8609be 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -34,7 +34,9 @@ from code_mower.context_contract import ContextError, ContextRequest from code_mower.context_store import ContextStore from test_context_connections import MemoryVault -from test_context_graph_query import PIN, git, graph_document, indexer, make_repository +from test_context_graph_query import ( + PIN, git, graph_document, indexer, make_repository, wide_graph_document, +) POLICY = { @@ -336,6 +338,64 @@ def test_reuse_after_a_rebuild_pauses_required_work(self) -> None: report, code = self.prepare(saved) self.assertEqual((code, report["status"]), (1, "required_unavailable")) + # -- a wide answer under the default document budget ------------------- + + def test_a_wide_answer_is_prepared_bounded_rather_than_refused_at_delivery(self) -> None: + """Six citable relationships, a budget of five, one usable packet. + + The default policy carries no ``max_documents``, so the contract's own + default of five applies -- and the contract enforces it when the packet + is loaded. A packet built to the query adapter's ceiling instead would + pass preparation and then fail every authorized load, which is a + required session paused over evidence the graph actually had. What the + builder must get is the bounded answer, told plainly that it is bounded. + """ + self.manifest = self.publish(wide_graph_document(6)) + record = self.record() + report, code = self.prepare(record) + self.assertEqual((code, report["status"], report["dependent_work"]), (0, "prepared", "usable")) + + saved = context_session.read(self.associations, record["session_id"]) + loaded = self.load(saved["packet"], "claude:builder") + packet = loaded.private_payload() + self.assertEqual(len(packet["documents"]), 5) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + self.assertEqual(packet["binding"]["generation"], self.manifest.generation) + # The builder reads it as evidence, not as a handle it cannot open. + evidence = context_delivery.render_evidence(loaded, saved["packet"]) + self.assertIn("example_pkg/config.py#L12", evidence) + + def test_replaying_a_bounded_packet_keeps_its_omissions_and_binding(self) -> None: + self.manifest = self.publish(wide_graph_document(6)) + record = self.record() + self.prepare(record) + saved = context_session.read(self.associations, record["session_id"]) + again, code = self.prepare(saved) + self.assertEqual((code, again["status"], again["reused"]), (0, "prepared", True)) + self.assertEqual( + context_session.read(self.associations, record["session_id"])["packet"], + saved["packet"], + ) + first = self.load(saved["packet"], "claude:builder").private_payload() + replayed = self.load(saved["packet"], "codex:builder").private_payload() + self.assertEqual(replayed["omissions"], first["omissions"]) + self.assertEqual(replayed["truncated"], first["truncated"]) + self.assertEqual(replayed["binding"], first["binding"]) + self.assertEqual(len(replayed["documents"]), 5) + + def test_an_optional_wide_answer_is_delivered_rather_than_degraded(self) -> None: + """A budget is not unavailability: optional work gets the evidence too.""" + self.manifest = self.publish(wide_graph_document(6)) + record = self.record(required=False) + report, code = self.prepare(record) + self.assertEqual((code, report["status"]), (0, "prepared")) + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual( + len(self.load(saved["packet"], "claude:builder").private_payload()["documents"]), 5, + ) + def _attach(self, handle: str, head: str): return context_delivery.reserve_attachment( self.store, "local-graph", handle, POLICY, @@ -462,6 +522,63 @@ def test_an_optional_consumer_at_another_revision_degrades_instead_of_pausing(se self.assertEqual((code, report["status"]), (0, "optional_unavailable")) self.assertEqual(list(self.private.glob(".p-*.json")), []) + def _publish_wide(self, callers: int = 6): + """Republish this checkout's graph with ``callers`` citable relationships.""" + self.manifest = lifecycle.build_graph( + self.repository, pin=PIN, indexer=indexer(wide_graph_document(callers)), + root=self.private, + ) + return self.manifest + + def test_a_wide_answer_is_delivered_bounded_instead_of_failing_validation(self) -> None: + """Six relationships, the contract's default budget of five, one packet. + + Before the document budget reached the traversal, the command built six + documents and the shared contract refused them on the way into the + store: ``packet_invalid``, required work paused, over evidence the graph + had and the policy simply did not have room for. The answer is five + documents that say they are five of more, and no validation failure. + """ + manifest = self._publish_wide(6) + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertNotEqual(report.get("reason"), "packet_invalid") + self.assertNotIn("failure_reason", report) + self.assertEqual(report["documents"], 5) + self.assertEqual(report["completeness"], "partial") + self.assertTrue(report["truncated"]) + + packet = context_packets.load_authorized( + self.store, "local-graph", report["packet_handle"], POLICY, + ContextRequest("owner/repo", "WORK-1", "claude:builder", manifest.commit), + ).private_payload() + self.assertEqual(len(packet["documents"]), 5) + self.assertIn("document_limit", packet["omissions"]) + self.assertEqual(packet["binding"]["generation"], manifest.generation) + # Every delivered citation still points at the immutable tree. + self.assertTrue(all( + citation["source"].startswith(("example_pkg/", "tests/")) + for item in packet["documents"] for citation in item["citations"] + )) + + def test_an_optional_wide_answer_is_delivered_rather_than_degraded(self) -> None: + """A budget is not unavailability: the optional caller gets evidence too.""" + self._publish_wide(6) + code, report = self.run_command(self.repository, required=False) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertNotEqual(report.get("reason"), "packet_invalid") + self.assertEqual(report["documents"], 5) + self.assertTrue(report["truncated"]) + + def test_an_answer_that_fits_the_budget_exactly_is_still_complete(self) -> None: + """The bound is only reported when it actually left evidence out.""" + self._publish_wide(5) + code, report = self.run_command(self.repository) + self.assertEqual((code, report["status"]), (0, "available")) + self.assertEqual(report["documents"], 5) + self.assertEqual(report["completeness"], "complete") + self.assertFalse(report["truncated"]) + def test_a_consumer_that_is_not_a_checkout_is_refused_rather_than_defaulted(self) -> None: """No revision at all is a refusal, not a fall back to the graph's.""" elsewhere = self.root / "not-a-checkout" diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index b81795da..a73198be 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -167,6 +167,35 @@ def graph_document(**extra) -> dict: } +def wide_graph_document(callers: int = 6, **extra) -> dict: + """``parse_config`` with ``callers`` distinct, separately citable callers. + + Every caller is a real node at its own line of an indexed file, so each + relationship the impact query reports carries two citations the bound commit + confirms. Nothing here is uncitable, duplicated, or dangling: the only + reason a document can go missing from a packet built over this graph is a + budget, which is what the tests using it are about. + """ + lines = SOURCES["example_pkg/loader.py"] + if callers + 1 > lines: # pragma: no cover - guards the fixture, not the code + raise AssertionError("the fixture file has no line left to cite") + return { + "nodes": [ + node("n-config", "parse_config", "example_pkg/config.py", 12), + *( + node(f"n-caller-{index}", f"caller_{index}", "example_pkg/loader.py", index + 1) + for index in range(1, callers + 1) + ), + ], + "edges": [edge(f"n-caller-{index}", "n-config", "calls") for index in range(1, callers + 1)], + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + "extracted_sources": sorted(SOURCES), + **extra, + } + + def node_link_document(**extra) -> dict: """The same graph as ``export.py::to_json`` writes it, for the clustered path. @@ -1184,6 +1213,78 @@ def test_packet_text_carries_no_indexed_content(self) -> None: self.assertIn(name, prose) +class DocumentBudgetTests(GraphWorkspace): + """The selected policy's document budget is what the packet is built to. + + ``MAX_DOCUMENTS`` is this adapter's own ceiling, and the shared contract + carries a separate, smaller default. A packet built to the ceiling alone is + not merely generous: ``context_contract`` refuses it at delivery, so the + answer a wide question deserves -- five documents and an honest + ``document_limit`` -- is instead no answer at all. These tests hold the two + budgets together and in the right direction: a policy may tighten what one + packet carries, never lift the ceiling. + """ + + def wide(self, callers: int = 6, **overrides) -> dict: + self.publish(wide_graph_document(callers)) + outcome = self.context(**overrides) + self.assertEqual(outcome.status, query.AVAILABLE) + return outcome.packet + + def test_the_default_policy_budget_bounds_a_wider_answer(self) -> None: + packet = self.wide() + self.assertEqual(len(packet["documents"]), 5) + self.assertLessEqual(len(packet["documents"]), contract.normalize_policy(policy())["max_documents"]) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_the_bounded_packet_is_the_deterministic_prefix_of_the_whole_answer(self) -> None: + """Nothing is reordered to fit: the budget cuts the tail, in place.""" + whole = self.wide(policy=policy(max_documents=6)) + bounded = self.wide(policy=policy(max_documents=5)) + self.assertEqual(len(whole["documents"]), 6) + self.assertEqual( + [item["text"] for item in bounded["documents"]], + [item["text"] for item in whole["documents"]][:5], + ) + + def test_an_answer_that_exactly_fits_its_budget_is_not_called_truncated(self) -> None: + """No eligible relationship was left out, so there is nothing to report.""" + packet = self.wide(policy=policy(max_documents=6)) + self.assertEqual(len(packet["documents"]), 6) + self.assertFalse(packet["truncated"]) + self.assertEqual(packet["completeness"], "complete") + self.assertNotIn("document_limit", packet["omissions"]) + + def test_a_budget_below_the_shared_default_bounds_the_packet_further(self) -> None: + """A policy may tighten past the contract's default, and is obeyed.""" + packet = self.wide(policy=policy(max_documents=2)) + self.assertEqual(len(packet["documents"]), 2) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_a_budget_above_the_adapters_ceiling_does_not_lift_it(self) -> None: + """The policy bounds the packet downward only; the ceiling still holds.""" + self.assertGreater(20, query.MAX_DOCUMENTS) + packet = self.wide(callers=20, policy=policy(max_documents=20)) + self.assertEqual(len(packet["documents"]), query.MAX_DOCUMENTS) + self.assertTrue(packet["truncated"]) + self.assertEqual(packet["completeness"], "partial") + self.assertIn("document_limit", packet["omissions"]) + + def test_the_summary_reports_the_same_bounded_count_as_the_packet(self) -> None: + """A metadata-only reader must not be told the answer was whole.""" + self.publish(wide_graph_document(6)) + outcome = self.context() + self.assertEqual(outcome.summary["documents"], len(outcome.packet["documents"])) + self.assertEqual(outcome.summary["documents"], 5) + self.assertTrue(outcome.summary["truncated"]) + self.assertEqual(outcome.summary["completeness"], "partial") + self.assertIn("document_limit", outcome.summary["omissions"]) + + class RecipientNeutralityTests(GraphWorkspace): """One approved packet, three recipients, no provider tools or credentials.""" From 1720b14a3b14aa83770be55c708e510905b7facf Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 14:06:58 -0700 Subject: [PATCH 22/33] Report evidence the depth limit left behind Reaching the requested traversal depth stopped expansion silently: only the relationship budget and the seed bound set `truncated`. For a dependency chain A -> B -> C -> D, the default question about A answered with C, reported `complete`, `truncated: false` and no omission -- so a recipient read "C depends on nothing" out of a graph that records C -> D. That contradicts the explicit traversal-completeness contract this module is built on. At the depth frontier the walk now asks what the node it is not expanding would have contributed, and says so when the answer is anything. Eligibility is measured exactly as the walk measures it: this question's direction and relationship filter, against relationship identities -- the provider's record, its endpoints and its wording -- that are not already in the answer. A boundary is not partial for being a boundary. So a chain that genuinely ends at the limit stays complete; a cycle or a `symbol` neighbourhood whose boundary edges were already stated from their other side stays complete; an already-reported self-loop or parallel pair stays complete; and a boundary edge that closes back onto a node already in the answer is still an omission, because its relationship is not. Direction and filter still decide what counts, node and relation budgets and edge deduplication are untouched, and the depth case joins the budget and seed cases under the existing provider_has_more/truncated/partial semantics rather than inventing a code the packet contract does not define. provider_partial keeps its own meaning: evidence the provider's document never carried, which no bound of ours cut. Regressions cover the finding's own A -> B -> C -> D case, an exact-boundary chain, an already-reported cycle, self-loop and parallel records, a boundary relationship back into the answer, direction and relationship-filter controls, and the packet and metadata-only summary carrying the omission with every retained citation still resolving against the bound commit and separable from a document-budget omission. One existing assertion changed. test_a_traversal_that_reaches_the_node_reports_partial proved two things at once over a symbol walk whose boundary really does have render calls load behind it. Its missing-endpoint half is unchanged; its "not truncation" half moved to an impact walk over the same fixture, whose boundary has no eligible relationship left, so it still proves strictly that a missing endpoint is partial without being truncated. The normalized-policy max_documents fix from the preceding head is untouched, and no policy default or internal cap is raised. Refs #914. Co-Authored-By: Claude Opus 5 (1M context) --- docs/context-graph-queries.md | 17 ++- src/code_mower/context_graph_query.py | 54 ++++++- tests/test_context_graph_query.py | 207 +++++++++++++++++++++++++- 3 files changed, 268 insertions(+), 10 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 38c1b758..6a2e2802 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -193,6 +193,20 @@ than the seed bound allows does the same: seeds the bound drops take their whole reachable neighbourhood out of the answer, so that is truncation too, not a complete result over the seeds that happened to sort first. +The depth limit is the third. A walk that stops at its requested depth on a node +that still has eligible relationships behind it has left evidence out, and +saying so is the whole point of the limit being explicit: for `a -> b -> c -> d` +a default `dependency` question about `a` ends at `c`, and a reader told that +answer is complete would conclude `c` depends on nothing. So the boundary sets +`truncated` and raises `provider_has_more`, exactly as the node budget does. + +It is not raised for reaching the boundary as such. Eligibility is measured the +way the walk measures it — this question's direction and relationship filter, +against relationships not already reported — so a chain that genuinely ends at +the boundary stays `complete`, and so does a cycle or a `symbol` neighbourhood +whose boundary edges were already stated from their other side. A parallel edge +the provider worded differently is a different relationship and does count. + Every reported relationship is the one edge the walk crossed, between that edge's own two endpoints. A second-hop result names the intermediate node and cites it — `render calls load (inferred, hop 2, reached from parse_config, …)` — @@ -210,7 +224,8 @@ reconverges keeps both edges into the node it reached twice; and a self-loop reached from both sides of a `symbol` neighbourhood, or an edge the provider recorded twice, is one relationship. Nothing about the bound changes: a relationship that does not fit the node budget still sets `truncated` and raises -`provider_has_more`, and the depth limit still applies. +`provider_has_more`, and the depth limit still applies and still reports what it +stopped. ## Citations are validated against the bound commit diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 905e461f..75dbc265 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -23,8 +23,10 @@ * **Queries are symbol-first, relationship-filtered and budgeted.** Default traversals in the evaluated provider returned 700-900 nodes and truncated silently. Every traversal here starts from named seeds, follows one filtered - relationship set, and stops at an explicit budget that is reported as - truncation rather than presented as a complete answer. + relationship set, and stops at an explicit budget or depth that is reported + as truncation rather than presented as a complete answer -- including when + what stopped it was the requested depth and the node it stopped on still had + eligible relationships behind it. * **Stale or unknown graph state is never answered from.** A required context request blocks; an optional one degrades to ordinary repository tools. There is no third outcome where a consumer is handed an older graph that looks @@ -884,7 +886,11 @@ def run_query( Node expansion and relationship reporting are bounded separately. A node is walked through once; a distinct directed relationship is reported once, including when both of its endpoints have already been seen. Anything left - out is left out by the budget or the depth limit, and says so. + out is left out by the budget or the depth limit, and says so: a depth + boundary that still has eligible, unreported relationships behind it sets + ``truncated`` exactly as the budget does, and one that has none -- a chain + that ends there, or a cycle whose boundary edges are already in the answer + -- leaves the result complete. """ if question not in QUESTIONS: raise ContextError("unsupported local graph question") @@ -926,9 +932,37 @@ def run_query( relations: list[Relation] = [] over_budget = False frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] + beyond_depth = False while frontier: node, seed, level = frontier.pop(0) if level >= limit: + # The requested depth stops the walk here, and stopping is allowed. + # Stopping *quietly* is not: for A -> B -> C -> D at depth 2 the + # answer ends at C, and a reader told the answer is complete would + # conclude C depends on nothing. So ask what this node would have + # contributed, and say so if the answer is anything. + # + # "Anything" is measured the same way the walk measures it: an + # eligible relationship under this question's direction and + # relationship filter whose identity is not already reported. That + # is what keeps the flag honest in both directions. A chain that + # genuinely ends at the boundary contributes nothing and stays + # complete; a cycle or a ``symbol`` neighbourhood whose boundary + # edges were already stated from the other side contributes + # nothing either, because those identities are already in the + # answer; and a parallel edge the provider worded differently is a + # different identity, so it counts. Reading adjacency costs no + # budget and reports nothing -- it only decides the flag. + # + # Ordering makes this exact rather than approximate: the frontier + # is FIFO and levels never decrease, so every node below the + # boundary has already been expanded by the time the first + # boundary node is popped. ``reported`` is final here. + if not beyond_depth and any( + (edge.source, edge.target, edge.relation, edge.kind, edge.evidence) not in reported + for edge, _ in _neighbours(graph, node.id, direction, kinds) + ): + beyond_depth = True continue for edge, other_id in _neighbours(graph, node.id, direction, kinds): # The provider's own record, endpoints and wording together: two @@ -961,10 +995,16 @@ def run_query( # reaches callers so that a test two hops away is found, but only the # tests are the answer. relations = [item for item in relations if item.node.kind == "test"] - # Two different ways to have left something out, reported as one state: a - # relationship budget that stopped the walk, and a seed bound that stopped - # it from ever starting at some of the target's definitions. - truncated = over_budget or seed_overflow + # Three different ways to have left something out, reported as one state: a + # relationship budget that stopped the walk, a seed bound that stopped it + # from ever starting at some of the target's definitions, and a depth limit + # that stopped it at a node with relationships nobody asked it to drop + # silently. All three are the same claim to a recipient -- this question has + # more evidence than this answer carries -- so all three raise + # ``provider_has_more`` rather than inventing a fourth code the packet + # contract does not define. ``provider_partial`` stays what it was: evidence + # the provider's document never carried at all, which no bound of ours cut. + truncated = over_budget or seed_overflow or beyond_depth if truncated: omissions.append("provider_has_more") if ambiguous or any(item.via.evidence == "ambiguous" for item in relations): diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index a73198be..1d533e81 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -821,6 +821,131 @@ def test_depth_still_bounds_a_walk_that_retains_relationships(self) -> None: self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) +class DepthBoundaryTests(unittest.TestCase): + """What the requested depth left behind, said out loud. + + Stopping at the requested depth is the contract. Stopping quietly is not: + for ``a -> b -> c -> d`` a default ``dependency`` question about ``a`` + answers with ``c`` and used to call that answer complete, so a reader + concluded ``c`` depends on nothing -- the graph's own record of ``c -> d`` + contradicting a packet that claimed to carry everything. + + The flag is not "the walk reached its depth". It is "the walk reached its + depth *and* left an eligible relationship unreported", measured the way the + walk measures eligibility: this question's direction and relationship + filter, against relationship identities not already in the answer. So these + hold both directions of that -- what must be reported, and what must not + become a false omission. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def chain(self, *, tail=()) -> query.CodeGraph: + """``alpha -> beta -> gamma``, plus whatever a test hangs off it.""" + return self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-c", "gamma", "example_pkg/report.py", 5), + node("n-d", "delta", "example_pkg/config.py", 30), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-c", "calls"), *tail], + ) + + def names(self, result: query.QueryResult) -> set: + return {item.node.name for item in result.relations} + + def test_a_chain_past_the_requested_depth_is_reported_as_truncation(self) -> None: + """The finding's own case: A -> B -> C -> D answered at depth 2.""" + graph = self.chain(tail=[edge("n-c", "n-d", "calls")]) + result = query.run_query(graph, question="dependency", target="alpha") + # The answer itself is unchanged -- the depth limit still bounds it. + self.assertEqual(self.names(result), {"beta", "gamma"}) + self.assertNotIn("delta", self.names(result)) + # What changed is that it no longer claims to be everything. + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + # Still not ``provider_partial``: the artifact carried ``c -> d`` in + # full. A bound of ours cut it, which is a different fact. + self.assertNotIn("provider_partial", result.omissions) + + def test_a_chain_that_ends_at_the_boundary_stays_complete(self) -> None: + """Exactly at the limit with nothing behind it: there is nothing to report.""" + result = query.run_query(self.chain(), question="dependency", target="alpha") + self.assertEqual(self.names(result), {"beta", "gamma"}) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_cycle_whose_boundary_edges_are_already_reported_stays_complete(self) -> None: + """Both records are in the answer, so the boundary omitted nothing.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-b", "calls"), edge("n-b", "n-a", "calls")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_an_already_reported_self_loop_and_parallel_pair_leave_it_complete(self) -> None: + """Identity, not endpoints: every record incident to the boundary is stated.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[edge("n-a", "n-a", "calls"), + edge("n-a", "n-b", "calls"), + edge("n-a", "n-b", "references")], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_boundary_relationship_back_into_the_answer_is_still_an_omission(self) -> None: + """Both endpoints are already reported nodes; the relationship is not. + + ``c -> a`` closes the cycle onto the seed, and its parallel twin says + something else about the same pair. Asking whether the boundary reaches + an *unseen node* would call this answer complete and drop two records + the graph carries, which is the identity confusion that produced the + earlier retained-relationship finding, one hop further out. + """ + graph = self.chain(tail=[edge("n-c", "n-a", "calls"), + edge("n-c", "n-a", "references")]) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + + def test_direction_decides_what_the_boundary_counts(self) -> None: + """One graph, two questions: the boundary edge points the wrong way for one.""" + graph = self.chain(tail=[edge("n-d", "n-c", "calls")]) + along = query.run_query(graph, question="dependency", target="alpha") + self.assertFalse(along.truncated) + self.assertNotIn("provider_has_more", along.omissions) + both = query.run_query(graph, question="symbol", target="alpha", depth=2) + self.assertTrue(both.truncated) + self.assertIn("provider_has_more", both.omissions) + + def test_the_relationship_filter_decides_what_the_boundary_counts(self) -> None: + """``contains`` normalizes to ``defines``, which ``dependency`` never walks.""" + graph = self.chain(tail=[edge("n-c", "n-d", "contains")]) + filtered = query.run_query(graph, question="dependency", target="alpha") + self.assertFalse(filtered.truncated) + self.assertNotIn("provider_has_more", filtered.omissions) + # ``symbol`` carries the whole vocabulary, so the same edge counts there. + neighbourhood = query.run_query(graph, question="symbol", target="alpha", depth=2) + self.assertTrue(neighbourhood.truncated) + self.assertIn("provider_has_more", neighbourhood.omissions) + + def test_an_unresolved_target_reports_no_depth_omission(self) -> None: + """Nothing was traversed, so the depth limit cut nothing.""" + result = query.run_query(self.chain(), question="dependency", target="no_such_symbol") + self.assertEqual(result.omissions, ("unresolved_entities",)) + self.assertFalse(result.truncated) + + def cyclic_graph_document() -> dict: """The shared fixture, plus the back-edge that makes the pair mutual. @@ -938,8 +1063,25 @@ def test_a_traversal_that_reaches_the_node_reports_partial(self) -> None: ) result = query.run_query(graph, question="symbol", target="parse_config") self.assertIn("provider_partial", result.omissions) - # Not truncation: no budget and no depth limit cut this. The evidence - # was never in the artifact. + + def test_a_missing_endpoint_is_not_itself_reported_as_truncation(self) -> None: + """No budget and no depth limit cut this; the artifact never carried it. + + The traversal is ``impact`` rather than the one-hop ``symbol`` + neighbourhood above because the two facts must not be read through each + other. In this fixture a ``symbol`` walk stops one hop out at ``load``, + which really does still have ``render calls load`` behind it -- genuine + depth truncation, and now reported as such. That would make a + ``assertFalse(truncated)`` here prove nothing about missing endpoints. + ``impact`` ends at ``render``, which has no eligible relationship left, + so the only thing this answer has left out is the endpoint the provider + never declared -- and that is partial without being truncated. + """ + graph = query.load_graph( + missing_endpoint_document(), generation="a" * 32, commit="b" * 40 + ) + result = query.run_query(graph, question="impact", target="parse_config") + self.assertIn("provider_partial", result.omissions) self.assertFalse(result.truncated) self.assertNotIn("provider_has_more", result.omissions) @@ -1285,6 +1427,67 @@ def test_the_summary_reports_the_same_bounded_count_as_the_packet(self) -> None: self.assertIn("document_limit", outcome.summary["omissions"]) +class DepthBoundaryPacketTests(GraphWorkspace): + """What a recipient reads when the depth limit left evidence behind. + + The traversal flag is only worth anything if it survives into the packet + and the metadata-only summary, and if a reader can tell *which* bound + spoke. A document budget and a depth limit are both "there is more", and + they are both true here at different times, so these hold them apart: the + depth case carries ``provider_has_more`` with no ``document_limit``, and + the evidence it did deliver is still fully cited. + """ + + def test_a_depth_bounded_answer_says_so_in_the_packet_and_the_summary(self) -> None: + # ``symbol`` is a one-hop neighbourhood of ``parse_config``, and the + # fixture puts ``render calls load`` one hop further out. + outcome = self.context(question="symbol") + self.assertEqual(outcome.status, query.AVAILABLE) + self.assertTrue(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "partial") + self.assertIn("provider_has_more", outcome.packet["omissions"]) + self.assertTrue(outcome.summary["truncated"]) + self.assertEqual(outcome.summary["completeness"], "partial") + self.assertIn("provider_has_more", outcome.summary["omissions"]) + + def test_the_depth_omission_is_not_reported_as_a_document_budget(self) -> None: + """Three documents against a budget of five: nothing was dropped to fit.""" + outcome = self.context(question="symbol") + self.assertEqual(len(outcome.packet["documents"]), 3) + self.assertLess( + len(outcome.packet["documents"]), + contract.normalize_policy(policy())["max_documents"], + ) + for code in ("document_limit", "provider_warning", "provider_partial"): + with self.subTest(code=code): + self.assertNotIn(code, outcome.packet["omissions"]) + + def test_the_retained_evidence_is_still_fully_cited_and_deliverable(self) -> None: + """Truncated is a statement about what is missing, not about what is there.""" + outcome = self.context(question="symbol") + report = context_graph.evaluate_graph_evidence( + outcome.packet, repository_root=self.repository, revision_state="matching", + ) + self.assertEqual(report.resolution_rate, 1.0) + self.assertTrue(report.meets_gate()) + validated = self.load( + outcome.packet, recipient="claude:builder", revision=self.manifest.commit) + self.assertEqual(validated.revision_state, "matching") + + def test_a_question_whose_walk_runs_out_first_is_still_complete(self) -> None: + """The same generation, a question that reaches the end of its evidence. + + ``impact`` stops at ``render``, which nothing calls, so its boundary + has no eligible relationship behind it. Without this the depth flag + could be satisfied by marking every answer partial, which would say + nothing at all. + """ + outcome = self.context() + self.assertFalse(outcome.packet["truncated"]) + self.assertEqual(outcome.packet["completeness"], "complete") + self.assertNotIn("provider_has_more", outcome.packet["omissions"]) + + class RecipientNeutralityTests(GraphWorkspace): """One approved packet, three recipients, no provider tools or credentials.""" From 1d3258a186adcacd3526bcf9774bc300ade3ed88 Mon Sep 17 00:00:00 2001 From: Jeff Huber Date: Mon, 14 Sep 2026 14:22:16 -0700 Subject: [PATCH 23/33] Keep canonically equivalent tracked paths apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git's tree holds exact bytes, so `src/café.py` spelled with U+00E9 and the same name spelled `e` + U+0301 are two entries, two blobs and two files that coexist in any Linux checkout. Completeness accounting folded both onto one NFC key -- in the manifest rows and in the pre-launch digest map -- while the eligible census still counted two. Give the pair identical bytes, fail one extraction and stamp the other, and the successful row overwrote the blank one: `complete`, `indexed_files: 2`, with one input's success standing in as proof for the other. That is exactly the coverage contract this module exists to refuse. Exact Git tree path identity now runs end to end. The denominator is the census path, and so is the digest map, so no input's expected bytes can be read from another's. Normalization survives only as what it was for: the macOS case where the copy's filesystem hands the provider a canonically equivalent spelling of the single name it was given. A manifest key spelled exactly as the tracked path is that input's record and is never reached past; the normalized fallback answers only when exactly one key folds onto it. Both ambiguities fail closed rather than resolving by position. An eligible input whose normal form is shared by any other tracked name is refused from the census itself -- immutable evidence taken before the launch, so a provider that wrote one row for the pair cannot hide that there were two -- and an input with several candidate rows and no exact one is refused as well. Neither is counted, each is reported as a count, and the run stays partial. Picking the first or the last colliding row would only be picking which failure to not report. Regressions cover the finding's own case in both census and manifest orders, colliding names with identical and with differing bytes, a manifest collapsed to one row over a collided census, missing and blank rows, a provider-only collision with an uncollided census, the legitimate single-path spelling mapping in both directions, exact-key precedence over a folded near-match, distinct non-ASCII names that do not fold together, and the digest map's exact keying. The dual-name cases use a census fixture, because macOS cannot hold both spellings at once. Classification, budgets, revision binding, containment and the pinned provider contract are untouched, and diagnostics stay counts only. Co-Authored-By: Claude Opus 5 (1M context) --- src/code_mower/context_graph_lifecycle.py | 107 +++++++++++- tests/test_context_graph_lifecycle.py | 194 ++++++++++++++++++++++ 2 files changed, 292 insertions(+), 9 deletions(-) diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py index b6850a08..0f638d91 100644 --- a/src/code_mower/context_graph_lifecycle.py +++ b/src/code_mower/context_graph_lifecycle.py @@ -2621,6 +2621,13 @@ def _materialized_digests( -- that case is already the partial one. An input the copy cannot be read for is simply absent from the map, and ``_read_completeness`` keeps the run partial for it rather than accepting the row unchecked. + + Keyed by the exact Git tree path, never by a folded spelling of it. Two + tracked names that differ only by Unicode normalization are two files with + two blobs, and a map keyed on their shared normal form holds one digest for + both -- so the expected bytes of one input would be read from the other's. + Normalization is a question about what the *provider* named its rows, and it + is asked once, in ``_read_completeness``, against this exact denominator. """ if census is None: return None @@ -2633,7 +2640,7 @@ def _materialized_digests( continue digest = _materialized_digest(host) if digest: - digests[unicodedata.normalize("NFC", path)] = digest + digests[path] = digest return digests @@ -2891,6 +2898,51 @@ def _provider_inputs( return _ProviderInputs(tuple(dispatched), frozenset(unsupported), tuple(unclassified)) +def _census_normalization_collisions(census: TrackedCensus) -> frozenset[str]: + """Normal forms that more than one distinct tracked path folds onto. + + Git's tree is a set of exact byte paths, and two of them can be canonically + equivalent without being equal: ``café.py`` spelled with U+00E9 and the same + name spelled ``e`` + U+0301 are two entries, two blobs and two files, and a + Linux checkout carries both at once. The manifest this build reads back is + keyed by name, so those two inputs are the one case where a name cannot + identify an input on its own. + + Taken from the whole census rather than from the dispatched subset, because + the collapse this exists to catch is a property of the tracked names, and an + eligible input colliding with a tracked file the pin would not dispatch is + the same ambiguity. The census is immutable evidence taken before the launch, + so a provider that folded the pair into one manifest row cannot hide that + they were two. Bounded by the census, which ``read_tracked_census`` has + already bounded. + """ + seen: dict[str, str] = {} + collided: set[str] = set() + for entry in census.entries: + folded = unicodedata.normalize("NFC", entry.path) + if seen.setdefault(folded, entry.path) != entry.path: + collided.add(folded) + return frozenset(collided) + + +def _manifest_keys_for( + path: str, rows: Mapping[str, Any], folded: Mapping[str, Sequence[str]] +) -> Sequence[str]: + """Which manifest keys could be a record of ``path``. + + Exact identity first: a key spelled exactly as the tracked path names one + input and no other, whatever else the manifest carries, so it is never + reached past. The normalized fallback exists for the ordinary macOS case, + where the copy's filesystem hands the provider a canonically equivalent + spelling of the one name it was given, and it answers only when exactly one + key folds onto that name -- more than one is two records this build cannot + attribute, and the caller keeps the run partial rather than picking one. + """ + if path in rows: + return (path,) + return folded.get(unicodedata.normalize("NFC", path), ()) + + def _read_completeness( manifest: Mapping[str, Any] | None, census: TrackedCensus | None, @@ -2954,6 +3006,22 @@ def _read_completeness( is unknown, one whose row disagrees with the bytes, and one whose zero-node result cannot be told apart from a failure. + Names, and the one place they are not identities. The denominator is keyed + by exact Git tree path, and so is the digest map: two tracked names that are + canonically equivalent without being equal are two entries with two blobs, + coexisting in any Linux checkout, and folding them together would let one + input's stamped row and expected digest answer for the other -- a blank row + for a failed extraction overwritten by a successful twin, reported as two + files indexed. Unicode normalization survives only as what it was for: the + macOS case where the copy's filesystem hands the provider a canonically + equivalent spelling of the single name it was given. So a manifest key that + matches an input exactly is that input's record, and the normalized fallback + answers only when exactly one key folds onto it and no other tracked name + shares its normal form (``_census_normalization_collisions``, + ``_manifest_keys_for``). Both ambiguities are counted and stay partial + rather than resolved by position, because picking the first or the last + colliding row is picking which failure to not report. + Blank rows are the cases the pin's rule makes blank: an extractor error or an anomalous zero-node extract. They stay partial here. The clean-room repeat that exited zero in 1.63 s requeued 54 entries; the retained @@ -2991,6 +3059,7 @@ def _read_completeness( inputs = _provider_inputs(census) eligible = inputs.dispatched rows: dict[str, Any] = {} + folded_keys: dict[str, list[str]] = {} malformed_rows = 0 for key, row in manifest.items(): if not isinstance(key, str): @@ -3001,19 +3070,30 @@ def _read_completeness( ): malformed_rows += 1 continue - rows[unicodedata.normalize("NFC", key)] = row - unsupported_keys = { - unicodedata.normalize("NFC", path) for path in inputs.unsupported - } + rows[key] = row + folded_keys.setdefault(unicodedata.normalize("NFC", key), []).append(key) + collisions = _census_normalization_collisions(census) missing = 0 unstamped = 0 unreadable = 0 mismatched = 0 processed = 0 unsupported = 0 + collided = 0 + ambiguous = 0 for path in eligible: - key = unicodedata.normalize("NFC", path) - row = rows.get(key) + if unicodedata.normalize("NFC", path) in collisions: + # Two tracked names this build cannot tell apart by name, and the + # manifest is keyed by name. Whichever row is found, one successful + # input would be standing in as proof for the other, so neither is + # counted and the run stays partial. + collided += 1 + continue + keys = _manifest_keys_for(path, rows, folded_keys) + if len(keys) > 1: + ambiguous += 1 + continue + row = rows[keys[0]] if keys else None if row is None: missing += 1 continue @@ -3021,7 +3101,7 @@ def _read_completeness( if not isinstance(digest, str) or not _MANIFEST_HASH.fullmatch(digest): unstamped += 1 continue - expected = digests.get(key) + expected = digests.get(path) if expected is None: # The row is well formed and this build cannot say what it should # have contained. Counting it would be believing the row on its own @@ -3029,7 +3109,7 @@ def _read_completeness( unreadable += 1 elif digest != expected: mismatched += 1 - elif key in unsupported_keys: + elif path in inputs.unsupported: # Read, not failed, and deterministically not extractable by this # pin. Counted on its own line rather than as a file this build # indexed, which it is not. @@ -3047,6 +3127,15 @@ def _read_completeness( notes.append(f"build could not re-read {unreadable} code files to check their hashes") if mismatched: notes.append(f"provider hashed {mismatched} code files that are not the bytes it was given") + if collided: + notes.append( + f"the census carried {collided} code files whose tracked names differ from " + "another tracked name only by Unicode normalization" + ) + if ambiguous: + notes.append( + f"provider manifest carried more than one candidate record for {ambiguous} code files" + ) if inputs.unclassified: notes.append( f"build could not classify {len(inputs.unclassified)} tracked inputs " diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py index fbaf1cb8..979835be 100644 --- a/tests/test_context_graph_lifecycle.py +++ b/tests/test_context_graph_lifecycle.py @@ -31,6 +31,7 @@ import tarfile import tempfile import time +import unicodedata import unittest import uuid from datetime import datetime, timezone @@ -1040,6 +1041,16 @@ def classify(prefix, **keywords): #: the one digest a finished manifest can carry for ``src/app.py``. CODE_INPUT_DIGEST = hashlib.md5(FIXTURE_INPUTS[CODE_INPUT], usedforsecurity=False).hexdigest() +#: The same rendered filename in its two canonically equivalent spellings: +#: ``é`` as U+00E9, and ``e`` followed by the U+0301 combining acute. Git holds +#: exact bytes, so these are two tracked entries with two blobs, and a Linux +#: checkout carries both at once. They are meant to render identically -- that +#: is the whole point -- so nothing here tells them apart by eye, and a census +#: fixture rather than a real checkout is the platform-independent way to test +#: the pair: macOS cannot hold both names at the same time. +NFC_INPUT = "src/café.py" +NFD_INPUT = "src/café.py" + def census_of(*paths: str) -> lifecycle.TrackedCensus: """A census naming ``paths``, shaped the way ``read_tracked_census`` does.""" @@ -1710,6 +1721,189 @@ def test_an_unsupported_extension_still_has_to_be_accounted_for(self) -> None: self.assertIn("does not account for 1 code files", " ".join(result.notes)) self.assertEqual(result.unsupported_inputs, 0) + def test_two_tracked_names_that_differ_only_by_normalization_stay_partial(self) -> None: + """One input's success must never stand in as proof for another's. + + ``src/café.py`` spelled with U+00E9 and the same name spelled ``e`` plus + U+0301 are two Git entries, two blobs and two files, and a Linux + checkout carries both at once. Give them identical bytes -- so their + expected digests are identical too and only the name can tell the rows + apart -- then fail one extraction and stamp the other. Folded onto a + shared key, the stamped row overwrites the blank one and both inputs + read as processed; held apart, neither is counted and the run is what it + actually was. Both census orders and both manifest orders, because a + fix that merely preferred the first or the last colliding row would pass + one of them. + """ + for census_order in ((NFC_INPUT, NFD_INPUT), (NFD_INPUT, NFC_INPUT)): + for rows_order in (census_order, census_order[::-1]): + with self.subTest(census=census_order, manifest=rows_order): + census = census_of(*census_order) + result = lifecycle._read_completeness( + { + rows_order[0]: manifest_row(""), + rows_order[1]: manifest_row(CODE_INPUT_DIGEST), + }, + census, + dict.fromkeys(census_order, CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("Unicode normalization", " ".join(result.notes)) + + def test_colliding_tracked_names_with_different_bytes_stay_partial(self) -> None: + """The digest map collapses the same way the rows do, so it is checked too. + + Different content gives the two inputs different expected digests, and a + map keyed on their shared normal form holds one of them for both. The + row that survives then agrees with whichever digest survived, which is a + comparison between two statements about one file and no statement at all + about the other. + """ + other = hashlib.md5(b"def other():\n return 1\n", usedforsecurity=False).hexdigest() + census = census_of(NFC_INPUT, NFD_INPUT) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), NFD_INPUT: manifest_row(other)}, + census, + {NFC_INPUT: CODE_INPUT_DIGEST, NFD_INPUT: other}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("Unicode normalization", " ".join(result.notes)) + + def test_a_collapsed_manifest_cannot_hide_a_collision_in_the_census(self) -> None: + """The census is the immutable evidence that there were two inputs. + + A provider that wrote one row for the pair leaves nothing in its own + output to say a second file existed. The denominator is not its output: + it is the tracked census taken before the launch, so the missing input + is visible whether or not the manifest admits to it. + """ + census = census_of(NFC_INPUT, NFD_INPUT) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, + dict.fromkeys((NFC_INPUT, NFD_INPUT), CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + + def test_one_tracked_name_still_matches_a_normalized_manifest_key(self) -> None: + """The macOS case normalization was for, which has to keep working. + + A single tracked name whose copy hands the provider a canonically + equivalent spelling is one input and one row. There is nothing to + confuse it with, so it matches and the run is complete -- in both + directions, because which spelling Git holds and which the filesystem + returns are independent. + """ + for tracked, written in ((NFD_INPUT, NFC_INPUT), (NFC_INPUT, NFD_INPUT)): + with self.subTest(tracked=tracked): + census = census_of(tracked) + result = lifecycle._read_completeness( + {written: manifest_row(CODE_INPUT_DIGEST)}, + census, + {tracked: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + + def test_a_manifest_key_spelled_exactly_is_that_input_s_own_record(self) -> None: + """Exact identity is never reached past for a folded near-match. + + Both spellings are present as keys and only one of them is this input's + name. The blank row beside it is about some other file, and a build that + resolved by normal form could read either one as the answer. + """ + census = census_of(NFD_INPUT) + digests = {NFD_INPUT: CODE_INPUT_DIGEST} + inputs = lifecycle._provider_inputs(census) + stamped = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(""), NFD_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, digests, inputs, + ) + self.assertEqual(stamped.completeness, lifecycle.COMPLETE) + self.assertEqual(stamped.indexed_files, 1) + blank = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), NFD_INPUT: manifest_row("")}, + census, digests, inputs, + ) + self.assertEqual(blank.completeness, lifecycle.PARTIAL) + self.assertEqual(blank.indexed_files, 0) + self.assertIn("unprocessed or requeued", " ".join(blank.notes)) + + def test_two_manifest_keys_folding_onto_one_input_are_not_resolved(self) -> None: + """A collision can be the provider's alone, and it is refused the same way. + + ``U+212B`` (ANGSTROM SIGN) and ``A`` plus ``U+030A`` both fold onto the + ``U+00C5`` the census holds, and neither is spelled the way the tracked + path is. Two candidate records and no exact one is two statements this + build cannot attribute to its single input. + """ + # Escaped, because all three names render identically and a reader + # cannot otherwise tell which line is which. + tracked = "src/Ångstrom.py" + legacy = "src/Ångstrom.py" + decomposed = "src/Ångstrom.py" + # The precondition this test is about, asserted rather than assumed. + self.assertEqual( + {unicodedata.normalize("NFC", key) for key in (legacy, decomposed)}, + {unicodedata.normalize("NFC", tracked)}, + ) + self.assertNotIn(tracked, (legacy, decomposed)) + census = census_of(tracked) + result = lifecycle._read_completeness( + {legacy: manifest_row(""), decomposed: manifest_row(CODE_INPUT_DIGEST)}, + census, + {tracked: CODE_INPUT_DIGEST}, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertEqual(result.indexed_files, 0) + self.assertIn("more than one candidate record", " ".join(result.notes)) + + def test_distinct_unicode_names_that_do_not_fold_together_are_unaffected(self) -> None: + """The refusal is about canonical equivalence, not about non-ASCII names.""" + naive = "src/naïve.py" + census = census_of(NFC_INPUT, naive) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST), naive: manifest_row(CODE_INPUT_DIGEST)}, + census, + dict.fromkeys((NFC_INPUT, naive), CODE_INPUT_DIGEST), + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 2) + + def test_input_digests_are_keyed_by_the_exact_tracked_path(self) -> None: + """What the copy was read for is recorded under the name Git holds. + + One file, one spelling, so this runs on any filesystem: the point is the + key, not a dual-name checkout. A map keyed on the normal form would + answer to a different tracked path than the one it was read for, and the + pre-launch digest is the only thing a manifest row is checked against. + The normalized manifest key beside it still resolves, which is the macOS + compatibility this keeps. + """ + root = self.root / "digests" + (root / "src").mkdir(parents=True) + (root / NFD_INPUT).write_bytes(FIXTURE_INPUTS[CODE_INPUT]) + census = census_of(NFD_INPUT) + digests = lifecycle._materialized_digests(root, census) + self.assertEqual(digests, {NFD_INPUT: CODE_INPUT_DIGEST}) + result = lifecycle._read_completeness( + {NFC_INPUT: manifest_row(CODE_INPUT_DIGEST)}, + census, + digests, + lifecycle._provider_inputs(census), + ) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 1) + def test_a_request_without_a_census_cannot_be_complete(self) -> None: """There is no denominator, so there is no coverage claim to make.""" self.assertEqual( From fef6faa6d851f93c51c87db71254f33f683f6f5d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 21:14:31 -0700 Subject: [PATCH 24/33] Bind published guided delivery to the actual consuming checkout (codex:917ad3b6181333ccfb92) deliver_session published path authorized repository-kind evidence against the remote PR head alone, never checking it against repo_path actual consuming revision. A builder checked out at commit B could receive evidence for commit A while the PR still pointed at A. Refuse closed when the checkout consuming revision cannot be resolved or does not match; organization and document connections are unaffected. Co-Authored-By: Claude Sonnet 5 --- src/code_mower/context_guided.py | 16 +++ tests/test_context_guided.py | 187 ++++++++++++++++++++++++++++++- 2 files changed, 202 insertions(+), 1 deletion(-) diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index 4c55407f..f24e9559 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any, Mapping +from . import context_graph_connection as graph_connection from . import context_review, context_session from .claude_audit_pr import _decision_authorities_for_repo from .context_contract import ContextError, ContextRequest @@ -350,6 +351,12 @@ def _builder_recipient(record: Mapping[str, Any]) -> str: return recipient +def _is_repository_connection(packet_store: ContextStore, name: str) -> bool: + """Whether ``name`` is a local repository graph rather than an organization connection.""" + with packet_store.locked(name) as locked: + return graph_connection.is_graph(locked.read()) + + def deliver_session( association_store: ContextStore, packet_store: ContextStore, @@ -385,6 +392,15 @@ def deliver_session( head, current = _remote_input( record["repo"], record["pr"], token=token, authorities=authorities, ) + if _is_repository_connection(packet_store, record["connection"]): + # This checkout's own consuming revision, not just the bound + # packet or PR head that ``deliver`` checks below; an + # unresolvable revision (e.g. not a Git checkout) fails closed. + if consuming_revision(repo_path) != head: + raise ContextError( + "repository context evidence is not bound to this " + "checkout's consuming revision" + ) text = deliver( packet_store, record["revision"], diff --git a/tests/test_context_guided.py b/tests/test_context_guided.py index 5dc7bb04..d2ec5162 100644 --- a/tests/test_context_guided.py +++ b/tests/test_context_guided.py @@ -9,12 +9,17 @@ from pathlib import Path from unittest import mock -from code_mower import context_audit, context_guided, context_session +from code_mower import context_audit, context_guided, context_prepare, context_session +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_contract import ContextError from code_mower.context_delivery import deliver, read_binding, save_feedback from code_mower.context_review import INPUT_HEADER from code_mower.context_store import ContextStore import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault +from test_context_graph_connection import POLICY as GRAPH_POLICY, RECIPIENTS as GRAPH_RECIPIENTS @unittest.skipUnless(os.name == "posix", "private storage needs POSIX") @@ -157,6 +162,36 @@ def test_builder_delivery_and_review_feedback_need_no_private_request_or_revisio context_session.read(self.associations, self.session["id"])["stage"], "reviewed" ) + def test_published_delivery_from_a_non_git_directory_succeeds_for_organization_evidence(self): + """Organization evidence never depends on a code revision (issue #982).""" + self.attach() + saved = context_session.read(self.associations, self.session["id"]) + with self.patches()[0], self.patches()[1], self.patches()[2]: + evidence = context_guided.deliver_session( + self.associations, self.fixture.store, saved, repo_path=self.root, + backend=self.fixture.backend, + ) + self.assertIn("Private evidence", evidence) + self.assertEqual( + context_session.read(self.associations, self.session["id"])["context_state"], "ready", + ) + + def test_published_delivery_still_refuses_a_moved_remote_head(self): + """The existing bound-packet/current-PR-head validation is untouched (issue #982).""" + self.attach() + saved = context_session.read(self.associations, self.session["id"]) + self.head = "d" * 40 + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.fixture.store, saved, repo_path=self.root, + backend=self.fixture.backend, + ) + self.assertEqual( + context_session.read(self.associations, self.session["id"])["context_state"], + "unavailable", + ) + def test_devin_host_builder_and_reviewer_use_the_common_packet(self): self.session.update(host="devin", orchestrator="devin", participants=[{"id": "claude"}, {"id": "devin"}]) @@ -461,5 +496,155 @@ def context_store(): self.assertEqual(self.fixture.backend.searches, 1) +@unittest.skipUnless(os.name == "posix", "private storage needs POSIX") +class GuidedRepositoryDeliveryTests(unittest.TestCase): + """Published repository evidence must match the actual consuming checkout (issue #982). + + Unlike ``GuidedContextTests``, the connection here is a real local + repository graph over a real throwaway Git checkout, so a mismatch + between the checkout doing the work and the bound PR revision is an + actual divergence between two resolvable commits, not a mocked value. + """ + + SESSION_ID = "a" * 32 + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.repository = graph_fixtures.make_repository(self.root) + self.private = self.root / "private" + self.private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + self.store = ContextStore(self.private, vault=MemoryVault()) + self.associations = context_session.association_store(self.private) + graph_connection.connect(self.store, "local-graph", { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": GRAPH_RECIPIENTS, + }) + selected = context_session.create( + self.associations, + { + "id": self.SESSION_ID, "repo": "owner/repo", "host": "codex", + "orchestrator": "codex", "participants": [{"id": "claude"}, {"id": "codex"}], + }, + work_item="WORK-1", + policy=GRAPH_POLICY, + ) + report, code = context_prepare.prepare( + self.associations, selected, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + self.record = context_session.read(self.associations, self.SESSION_ID) + self.head = self.manifest.commit + self.comments: list[dict] = [] + + def _pull(self, *_args, **_kwargs): + return {"head": {"sha": self.head}} + + def _comments(self, *_args, **_kwargs): + return list(self.comments) + + def _github(self, _method, _path, **_kwargs): + return {} + + def _post(self, _repo, _pr, body, **_kwargs): + self.comments.append( + {"id": len(self.comments) + 1, "user": {"login": "controller"}, "body": body} + ) + return {"id": len(self.comments)} + + def patches(self): + return ( + mock.patch.object( + context_guided, "_github_access", return_value=("token", ("controller",)), + ), + mock.patch.object(context_guided, "fetch_pull_request", side_effect=self._pull), + mock.patch.object(context_guided, "fetch_issue_comments", side_effect=self._comments), + mock.patch.object(context_guided, "_gh_request", side_effect=self._github), + mock.patch.object(context_guided, "post_pr_comment", side_effect=self._post), + ) + + def attach(self, pr=1): + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + ): + current = context_session.read(self.associations, self.SESSION_ID) + return context_guided.attach_session( + self.associations, self.store, current, repo_path=self.repository, + pr=pr, backend=None, + ) + + def _advance_the_checkout(self): + """Commit new work so the checkout's HEAD moves off the graph's commit.""" + (self.repository / "example_pkg" / "config.py").write_text("changed\n", encoding="utf-8") + graph_fixtures.git(self.repository, "add", ".") + graph_fixtures.git(self.repository, "commit", "-q", "-m", "second") + + def test_repository_delivery_matches_the_actual_consuming_checkout(self): + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + published = context_session.read(self.associations, self.SESSION_ID) + with self.patches()[0], self.patches()[1], self.patches()[2]: + evidence = context_guided.deliver_session( + self.associations, self.store, published, repo_path=self.repository, backend=None, + ) + self.assertIn("Private evidence", evidence) + self.assertIn("example_pkg/config.py#L12", evidence) + self.assertEqual( + context_session.read(self.associations, self.SESSION_ID)["context_state"], "ready", + ) + + def test_published_delivery_refuses_when_the_actual_checkout_has_moved(self): + self.attach() + published = context_session.read(self.associations, self.SESSION_ID) + # The remote PR head the stub reports is untouched; only the actual + # consuming checkout moves, which is exactly the divergence issue + # #982 describes. + self._advance_the_checkout() + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, published, + repo_path=self.repository, backend=None, + ) + after = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(after["context_state"], "unavailable") + self.assertEqual(after["stage"], "attached") + + def test_published_repository_delivery_from_a_non_git_directory_fails_closed(self): + self.attach() + published = context_session.read(self.associations, self.SESSION_ID) + outside = self.root / "not-a-checkout" + outside.mkdir() + with self.patches()[0], self.patches()[1], self.patches()[2]: + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, published, repo_path=outside, backend=None, + ) + self.assertEqual( + context_session.read(self.associations, self.SESSION_ID)["context_state"], + "unavailable", + ) + + def test_unpublished_repository_delivery_still_binds_the_consuming_checkout(self): + evidence = context_guided.deliver_session( + self.associations, self.store, self.record, repo_path=self.repository, backend=None, + ) + self.assertIn("Private evidence", evidence) + self._advance_the_checkout() + moved = context_session.read(self.associations, self.SESSION_ID) + with self.assertRaises(ContextError): + context_guided.deliver_session( + self.associations, self.store, moved, repo_path=self.repository, backend=None, + ) + + if __name__ == "__main__": unittest.main() From c75ea404a6ba6be44ca625e321e4264bbb13e7d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 21:53:29 -0700 Subject: [PATCH 25/33] Bind attached replay to the consuming checkout (codex:b7f5dbb1412eb89a3797) Standalone attached replay never bound repository evidence to the actual consuming checkout: context_command.py obtained the remote PR head and called deliver() without the local consuming revision, and the shared _packet_for_binding() defaulted the consuming revision to that head. A checkout at a different commit, or a non-Git directory, could therefore receive repository evidence authorized for a revision it never held. deliver() now takes an explicit consuming_revision, and the shared helper no longer substitutes the attachment/PR head when it is missing; a caller that cannot name it fails closed instead. The standalone CLI derives it from --repo-path, guided delivery and feedback pass the actual checkout revision, and independent audit preparation and finish pass the immutable audited target head rather than a control checkout's revision. Co-Authored-By: Claude Sonnet 5 --- src/code_mower/context_audit.py | 12 +++- src/code_mower/context_command.py | 7 ++- src/code_mower/context_delivery.py | 23 +++++--- src/code_mower/context_guided.py | 6 ++ tests/test_context_audit.py | 70 +++++++++++++++++++++++- tests/test_context_command.py | 88 ++++++++++++++++++++++++++++++ tests/test_context_delivery.py | 62 +++++++++++++++++++++ 7 files changed, 257 insertions(+), 11 deletions(-) diff --git a/src/code_mower/context_audit.py b/src/code_mower/context_audit.py index 067272bc..dedc7f25 100644 --- a/src/code_mower/context_audit.py +++ b/src/code_mower/context_audit.py @@ -69,8 +69,12 @@ def finish(self, *, head, prose): if not review_matches(marker(current, review=True), current, head=head): return False if self.delivery is not None: + # The immutable audited target head, not a control checkout's + # revision: the checkout running this audit may differ from + # the code being reviewed. verified = deliver(self.store, current["revision"], repository=self.repository, - pr=self.pr, head=head, recipient=self.recipient, current=current, backend=self.backend) + pr=self.pr, head=head, recipient=self.recipient, current=current, + consuming_revision=head, backend=self.backend) if verified.text != self.delivery.text: return False save_feedback(self.store, verified, self.recipient.split(":")[0], prose) @@ -109,8 +113,12 @@ def prepare(*, repository, pr, head, host, authorities, fetch_comments, state.ready = review_matches(marker(current, review=True), current, head=head) return state state.store = store if store is not None else ContextStore(state_dir) + # ``head`` is the immutable review-target revision this audit is for, + # never derived from ``repo_path`` or ``Path.cwd()``: the audit + # control checkout may differ from the selected review target. state.delivery = deliver(state.store, current["revision"], repository=repository, - pr=pr, head=head, recipient=state.recipient, current=current, backend=backend) + pr=pr, head=head, recipient=state.recipient, current=current, + consuming_revision=head, backend=backend) state.ready = True except (ContextError, OSError, ValueError, RuntimeError, TypeError): pass diff --git a/src/code_mower/context_command.py b/src/code_mower/context_command.py index 8a220dfd..1cf8ba0c 100644 --- a/src/code_mower/context_command.py +++ b/src/code_mower/context_command.py @@ -121,8 +121,13 @@ def publish(metadata): if current is None: raise ContextError("no trusted current context input is declared") head = fetch_pull_request(binding["repository"], binding["pr"], token=token)["head"]["sha"] + # The actual consuming checkout, derived from --repo-path, not the + # remote PR head: a checkout at a different commit, or a directory + # that is not a Git checkout at all, must fail before any evidence or + # feedback is produced. delivery = deliver(store, args.revision, repository=binding["repository"], pr=binding["pr"], head=head, - recipient=args.recipient, current=current) + recipient=args.recipient, current=current, + consuming_revision=consuming_revision(args.repo_path)) if args.command == "feedback": feedback = delivery.binding["feedback"].get(args.reviewer) if feedback is None: diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index 543dfdb4..25e31492 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -79,11 +79,11 @@ def read_binding(store, revision): def _packet_for_binding(store, binding, recipient, *, backend=None, revision=None): if recipient not in SUPPORTED_RECIPIENTS: raise ContextError("this participant cannot consume private context in this release") - # The consuming revision of a delivery is the head the binding was published - # for, which the caller has already confirmed against the trusted current - # input. A repository-kind connection re-derives its authorization from that - # commit; an organization connection ignores it. - revision = revision or binding["metadata"]["head"] + # ``revision`` is the caller's actual consuming checkout revision, or an + # already-verified immutable review-target head; it is never defaulted to + # the head this binding happens to have been published for. A repository + # connection re-derives its authorization from exactly that commit and + # fails closed when it is missing; an organization connection ignores it. packet = load_authorized(store, binding["connection"], binding["handle"], binding["policy"], ContextRequest(binding["repository"], binding["work_item"], recipient), backend=backend, revision=revision) @@ -255,7 +255,16 @@ class Delivery: binding: dict = field(repr=False) -def deliver(store, revision, *, repository, pr, head, recipient, current, backend=None): +def deliver(store, revision, *, repository, pr, head, recipient, current, consuming_revision=None, backend=None): + """Replay one published binding for an approved recipient. + + ``consuming_revision`` is the caller's actual consuming checkout revision, + or an already-verified immutable review-target head; ``None`` means the + caller cannot name one. It is never defaulted to ``head`` here: a caller + that knows only the trusted remote head, not the local checkout doing the + work, must say so explicitly rather than let a repository-kind connection + be silently authorized against a revision it never held. + """ binding = read_binding(store, revision) try: current = context_review.validate(dict(current)) @@ -266,7 +275,7 @@ def deliver(store, revision, *, repository, pr, head, recipient, current, backen raise ContextError("context input is missing, unpublished, or no longer current") if not context_review.review_matches(context_review.marker(current, review=True), current, head=head): raise ContextError("context input is unavailable or expired") - packet = _packet_for_binding(store, binding, recipient, backend=backend) + packet = _packet_for_binding(store, binding, recipient, backend=backend, revision=consuming_revision) return Delivery(dict(current), render_evidence(packet, binding["handle"]), binding) diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index f24e9559..567d7d56 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -229,6 +229,10 @@ def attach_session( head=head, recipient=record["host"] + ":orchestrator", current=current, + # This checkout's actual consuming revision, not the + # remote head alone; a repository connection fails + # closed if the checkout has moved. + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: @@ -409,6 +413,7 @@ def deliver_session( head=head, recipient=recipient, current=current, + consuming_revision=consuming_revision(repo_path), backend=backend, ).text except ContextError as exc: @@ -458,6 +463,7 @@ def feedback_session( head=head, recipient=recipient, current=current, + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: diff --git a/tests/test_context_audit.py b/tests/test_context_audit.py index e296aace..38930ad4 100644 --- a/tests/test_context_audit.py +++ b/tests/test_context_audit.py @@ -5,17 +5,25 @@ import json import os import subprocess +import tempfile import unittest from contextlib import ExitStack, redirect_stderr +from pathlib import Path from unittest import mock from code_mower import claude_audit_pr as claude from code_mower import codex_audit_pr as codex -from code_mower import context_audit, context_packets +from code_mower import context_audit, context_delivery, context_packets +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle +from code_mower.context_contract import ContextRequest from code_mower.context_delivery import read_binding from code_mower.context_review import INPUT_HEADER, marker +from code_mower.context_store import ContextStore from code_mower.provider_runners import verdict_artifacts import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -193,3 +201,63 @@ def test_context_bound_artifact_cannot_be_reposted_without_fresh_audit(self): with self.assertRaisesRegex(ValueError, 'fresh authorized audit'): verdict_artifacts.repost_audit_verdict_artifact(result.verdict_artifact_path, token='test-authorization') post.assert_not_called() + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphIndependentReviewTargetTests(unittest.TestCase): + """Independent review replay binds to the immutable audited head, never to + ``repo_path`` or a control checkout (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.head = self.manifest.commit + recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': recipients, + }) + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + result = context_packets.fetch(self.store, 'local-graph', spec, revision=self.head) + self.current = context_delivery.attach(self.store, 'local-graph', result['packet_handle'], + self.policy, ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, + publish=lambda metadata: None) + self.comments = [{'user': {'login': 'controller'}, 'body': INPUT_HEADER + '\n\n' + marker(self.current)}] + self.root = root + + def prepare(self, repo_path): + return context_audit.prepare( + repository='owner/repo', pr=42, head=self.head, host='codex', authorities=('controller',), + fetch_comments=lambda: self.comments, store=self.store, repo_path=repo_path, + ) + + def test_prepare_authorizes_against_the_review_target_head_not_a_foreign_control_checkout(self): + outside = self.root / 'unrelated-control-checkout' + outside.mkdir() + state = self.prepare(outside) + self.assertTrue(state.ready) + self.assertIn('Private evidence', state.text) + + def test_finish_verifies_against_the_review_target_head_not_a_foreign_control_checkout(self): + outside = self.root / 'another-unrelated-control-checkout' + outside.mkdir() + state = self.prepare(outside) + self.assertTrue(state.ready) + prose = 'Summary: preserved the routing constraint. No blocking regressions found.' + self.assertTrue(state.finish(head=self.head, prose=prose)) + self.assertEqual( + read_binding(self.store, self.current['revision'])['feedback']['codex'], prose, + ) diff --git a/tests/test_context_command.py b/tests/test_context_command.py index 43502b55..4954e33a 100644 --- a/tests/test_context_command.py +++ b/tests/test_context_command.py @@ -11,10 +11,16 @@ from unittest import mock from code_mower import context_command as command, context_packets, work_orders +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_contract import ContextError from code_mower.context_delivery import read_binding from code_mower.context_review import INPUT_HEADER, marker, parse +from code_mower.context_store import ContextStore import test_context_delivery as fixtures +import test_context_graph_query as graph_fixtures +from test_context_connections import MemoryVault +from test_context_graph_connection import POLICY as GRAPH_POLICY, RECIPIENTS as GRAPH_RECIPIENTS @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -165,6 +171,88 @@ def test_revision_cannot_silently_ignore_a_different_explicit_connection(self): self.assertEqual((code, out, credentials), (1, '', 0)) +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphContextCommandTests(unittest.TestCase): + """Standalone attached replay must bind repository evidence to the actual + consuming checkout named by ``--repo-path``, never to the remote PR head + alone (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': GRAPH_RECIPIENTS, + }) + self.head = self.manifest.commit + self.comments = [] + + def invoke(self, args, spec=None, *, actor='controller'): + def github(method, path, **kwargs): + if path == '/user': + return {'login': actor} + return {} + def publish(repo, pr, body, **kwargs): + self.comments.append({'user': {'login': actor}, 'body': body}) + return {'id': 123} + with ExitStack() as stack: + stdout = stack.enter_context(redirect_stdout(io.StringIO())) + stderr = stack.enter_context(redirect_stderr(io.StringIO())) + stack.enter_context(mock.patch.object(command, 'ContextStore', return_value=self.store)) + stack.enter_context(mock.patch.object(command, '_decision_authorities_for_repo', return_value=('controller',))) + stack.enter_context(mock.patch.object(command, '_github', return_value='test-authorization')) + stack.enter_context(mock.patch.object(command, '_gh_request', side_effect=github)) + stack.enter_context(mock.patch.object(command, 'post_pr_comment', side_effect=publish)) + stack.enter_context(mock.patch.object(command, 'fetch_pull_request', return_value={'head': {'sha': self.head}})) + stack.enter_context(mock.patch.object(command, 'fetch_issue_comments', return_value=self.comments)) + stack.enter_context(mock.patch.object(command.sys, 'stdin', SimpleNamespace(buffer=io.BytesIO(json.dumps(spec).encode())))) + code = work_orders.context_main(args) + return code, stdout.getvalue(), stderr.getvalue() + + def attach(self): + packet_spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': GRAPH_POLICY} + result = context_packets.fetch(self.store, 'local-graph', packet_spec, revision=self.head) + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'pr': 7, 'policy': GRAPH_POLICY, + 'packet': result['packet_handle']} + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin'], spec) + self.assertEqual(code, 0, err) + return json.loads(out)['revision'] + + def test_replay_succeeds_from_the_actual_consuming_checkout(self): + revision = self.attach() + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(self.repository)]) + self.assertEqual(code, 0, err) + self.assertIn('Private evidence', out) + + def test_replay_refuses_a_checkout_that_moved_off_the_attachment_head(self): + """Checkout B must not receive evidence bound to checkout A's head.""" + revision = self.attach() + graph_fixtures.git(self.repository, 'commit', '--allow-empty', '-q', '-m', 'checkout moved to B') + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(self.repository)]) + self.assertEqual((code, out), (1, '')) + + def test_replay_refuses_a_non_git_directory(self): + revision = self.attach() + outside = self.repository.parent / 'not-a-checkout' + outside.mkdir() + code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', + '--repo-path', str(outside)]) + self.assertEqual((code, out), (1, '')) + + class ContextWorkOrderTests(unittest.TestCase): def test_work_order_tracks_only_opaque_packet_and_does_not_expand_cloud_metadata(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/tests/test_context_delivery.py b/tests/test_context_delivery.py index 2fa6d446..c17da5e8 100644 --- a/tests/test_context_delivery.py +++ b/tests/test_context_delivery.py @@ -7,6 +7,8 @@ import unittest from pathlib import Path +from code_mower import context_graph_connection as graph_connection +from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_connections import connect, disconnect from code_mower.context_contract import ContextError, ContextRequest, load_packet from code_mower.context_delivery import (SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, attach, deliver, public_verdict, @@ -16,6 +18,7 @@ from test_context_connections import MemoryVault from test_context_packets import RetrievalBackend from test_coworker_retrieval import POLICY +import test_context_graph_query as graph_fixtures @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') @@ -113,6 +116,11 @@ def test_retrieval_refresh_deletes_old_binding_and_feedback(self): self.delivery(current) self.assertEqual(list(self.store.root.glob('.d-*.json')), []) + def test_organization_replay_succeeds_without_a_consuming_revision(self): + """Organization evidence has no code revision (codex:b7f5dbb1412eb89a3797).""" + current = self.attach() + self.assertTrue(self.delivery(current, consuming_revision=None).text) + def test_public_verdict_has_only_metadata_and_never_model_authored_findings(self): current = self.attach() delivery = self.delivery(current) @@ -162,3 +170,57 @@ def test_local_repository_graph_uses_common_renderer_without_oauth_identity(self self.assertIn('Synthetic evidence: parser calls validator.', texts[0]) self.assertNotIn('/example/repository', texts[0]) self.assertNotIn('principal', texts[0]) + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphDeliveryBindingTests(unittest.TestCase): + """A repository replay must bind to its explicit consuming revision, never the + attachment/PR head it happens to have been published for (codex:b7f5dbb1412eb89a3797).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': self.recipients, + }) + self.head = self.manifest.commit # checkout A + self.other = 'b' * 40 # a different checkout, B + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + result = fetch(self.store, 'local-graph', spec, revision=self.head) + self.current = attach(self.store, 'local-graph', result['packet_handle'], self.policy, + ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, + publish=lambda metadata: None) + + def delivery(self, **kwargs): + return deliver(self.store, self.current['revision'], repository='owner/repo', pr=42, head=self.head, + recipient='codex:builder', current=self.current, **kwargs) + + def test_replay_matches_the_actual_consuming_revision(self): + self.assertIn('Private evidence', self.delivery(consuming_revision=self.head).text) + + def test_replay_refuses_a_different_consuming_revision(self): + """Checkout B must not receive evidence authorized for checkout A.""" + with self.assertRaises(ContextError): + self.delivery(consuming_revision=self.other) + + def test_replay_never_silently_substitutes_the_attachment_head(self): + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + with self.assertRaises(ContextError): + self.delivery() + with self.assertRaises(ContextError): + self.delivery(consuming_revision=None) From 135a51f966606feaaa238bb085c6251e0242b56c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 22:29:46 -0700 Subject: [PATCH 26/33] Bind fresh attachments to the consuming checkout (codex:65a17212478a56416b1c) reserve_attachment() reconstructed the caller's request with the remote PR head and supplied that same head to authorization as the consuming revision, replacing an unknown or different caller checkout. A standalone or guided caller could therefore reserve and publish repository evidence for the PR head while actually running from a different checkout or a non-Git directory. reserve_attachment() and attach() now take an explicit consuming_revision, separate from the PR head kept as attachment metadata; it is never substituted from head, and a repository connection fails closed when it is missing. The standalone CLI derives it from --repo-path, and guided attachment reservation passes the actual checkout revision for both a fresh reservation and a pending/uncertain retry, so a retry revalidates the current consumer before marking an attachment published. Co-Authored-By: Claude Sonnet 5 --- src/code_mower/context_command.py | 7 ++- src/code_mower/context_delivery.py | 35 +++++++++----- src/code_mower/context_guided.py | 9 +++- tests/test_context_audit.py | 6 ++- tests/test_context_command.py | 59 +++++++++++++++++++++-- tests/test_context_delivery.py | 66 +++++++++++++++++++++++++- tests/test_context_graph_connection.py | 35 ++++++++++---- tests/test_context_guided.py | 37 +++++++++++++++ 8 files changed, 225 insertions(+), 29 deletions(-) diff --git a/src/code_mower/context_command.py b/src/code_mower/context_command.py index 1cf8ba0c..87175104 100644 --- a/src/code_mower/context_command.py +++ b/src/code_mower/context_command.py @@ -108,9 +108,14 @@ def publish(metadata): # The current trusted gate independently rejects policy downgrade. publish(metadata) else: + # The actual consuming checkout, derived from --repo-path, not + # the remote PR head: a checkout at a different commit, or a + # directory that is not a Git checkout at all, must fail + # before any evidence is reserved or published. metadata = attach(store, args.connection, spec["packet"], spec["policy"], ContextRequest(spec["repository"], spec["work_item"], args.host + ":orchestrator"), - pr=spec["pr"], head=head, publish=publish) + pr=spec["pr"], head=head, publish=publish, + consuming_revision=consuming_revision(args.repo_path)) print(json.dumps({"status": "attached", **metadata}, sort_keys=True)) return 0 if not args.revision or args.packet: diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index 25e31492..30d1a035 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -102,6 +102,7 @@ def reserve_attachment( pr, head, revision=None, + consuming_revision=None, backend=None, ): """Reserve one unpublished binding before any remote publication. @@ -110,19 +111,26 @@ def reserve_attachment( touching GitHub and resume that exact intent after a crash. Repeating the same reservation is idempotent; a conflicting reuse fails closed. - ``revision`` here is that attachment handle, not a Git revision. The Git - revision an attachment binds is ``head``: the PR head the caller read from - the trusted remote. Repository-kind evidence is authorized and checked - against that commit, so a packet prepared while the checkout sat at commit A - cannot be attached to a PR whose head is commit B. + ``revision`` here is that attachment handle, not a Git revision. ``head`` + is attachment metadata: the PR head the caller read from the trusted + remote. ``consuming_revision`` is the caller's actual consuming checkout + revision, or ``None`` when the caller cannot name one; it is never + defaulted to ``head``, because a caller that knows only the trusted + remote head, not the local checkout doing the work, must say so + explicitly rather than let a repository-kind connection be silently + authorized against a revision it never held. Repository-kind evidence is + authorized against that consuming revision and separately checked against + ``head``, so a packet prepared while the checkout sat at commit A cannot + be attached to a PR whose head is commit B, and a checkout that has + itself moved to commit B cannot attach evidence for commit A. """ if request.recipient not in SUPPORTED_RECIPIENTS or not request.recipient.endswith(":orchestrator"): raise ContextError("an approved orchestrator must attach context") policy = normalize_policy(policy) packet = load_authorized( store, name, handle, policy, - ContextRequest(request.repository, request.work_item, request.recipient, head), - backend=backend, revision=head, + ContextRequest(request.repository, request.work_item, request.recipient, consuming_revision), + backend=backend, revision=consuming_revision, ) payload = packet.private_payload() revision = revision or uuid.uuid4().hex @@ -137,8 +145,10 @@ def reserve_attachment( # The local graph's "authorization changed" is a rebuild: the # published generation is what a packet binds, so a graph rebuilt # between preparation and attachment fails the same check a revoked - # organization authorization does. - generation = graph_connection.current_generation(state, root=store.root, revision=head) + # organization authorization does. ``consuming_revision`` cannot be + # ``None`` here: a graph connection already refused the load above + # when it was missing. + generation = graph_connection.current_generation(state, root=store.root, revision=consuming_revision) # Stated here as well as enforced on the load, because this is the # line an attachment is read off: repository evidence describes one # commit's code, and the commit this PR is at is the only one it may @@ -227,14 +237,17 @@ def retire_attachment(store, name, handle, revision): _remove_attachment(store, name, handle, revision, published=True) -def attach(store, name, handle, policy, request: ContextRequest, *, pr, head, publish, backend=None): +def attach(store, name, handle, policy, request: ContextRequest, *, pr, head, publish, + consuming_revision=None, backend=None): """Publish a new input revision before any participant can use that binding. ``publish`` is a trusted runtime callback for the selected repository/PR, never provider code. A failed publication leaves the local binding unusable. + ``consuming_revision`` is forwarded to ``reserve_attachment`` unchanged. """ metadata = reserve_attachment( - store, name, handle, policy, request, pr=pr, head=head, backend=backend, + store, name, handle, policy, request, pr=pr, head=head, + consuming_revision=consuming_revision, backend=backend, ) revision = metadata["revision"] try: diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index 567d7d56..f5faa82e 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -112,6 +112,7 @@ def _reserve_for_record( packet_store: ContextStore, record: Mapping[str, Any], *, + repo_path: Path, backend: Any, ) -> dict[str, Any]: try: @@ -126,6 +127,10 @@ def _reserve_for_record( pr=record["pr"], head=record["head"], revision=record["revision"], + # This checkout's actual consuming revision, not the remote head + # alone; a repository connection fails closed if the checkout has + # moved, both for a fresh reservation and a pending/uncertain retry. + consuming_revision=consuming_revision(repo_path), backend=backend, ) except ContextError as exc: @@ -245,7 +250,7 @@ def attach_session( if record["pr"] != pr: raise ContextError("a saved attachment intent targets a different pull request") metadata = _reserve_for_record( - association_store, packet_store, record, backend=backend, + association_store, packet_store, record, repo_path=repo_path, backend=backend, ) if current == metadata: if record["head"] == head: @@ -290,7 +295,7 @@ def attach_session( }, ) metadata = _reserve_for_record( - association_store, packet_store, record, backend=backend, + association_store, packet_store, record, repo_path=repo_path, backend=backend, ) return _finish_publication( association_store, diff --git a/tests/test_context_audit.py b/tests/test_context_audit.py index 38930ad4..57814ff7 100644 --- a/tests/test_context_audit.py +++ b/tests/test_context_audit.py @@ -232,9 +232,13 @@ def setUp(self): spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} result = context_packets.fetch(self.store, 'local-graph', spec, revision=self.head) + # The fixture's own actual revision -- this checkout is genuinely at + # ``self.head`` when the packet is attached -- never the immutable + # review-target head that ``context_audit`` is separately exercised + # against below. self.current = context_delivery.attach(self.store, 'local-graph', result['packet_handle'], self.policy, ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, - publish=lambda metadata: None) + publish=lambda metadata: None, consuming_revision=self.head) self.comments = [{'user': {'login': 'controller'}, 'body': INPUT_HEADER + '\n\n' + marker(self.current)}] self.root = root diff --git a/tests/test_context_command.py b/tests/test_context_command.py index 4954e33a..4b5087a2 100644 --- a/tests/test_context_command.py +++ b/tests/test_context_command.py @@ -88,6 +88,15 @@ def test_attach_publishes_pending_before_content_free_input_and_delivers_same_pa self.assertEqual(text, texts[0]) self.assertEqual(self.fixture.backend.searches, 1) + def test_attach_succeeds_for_organization_evidence_from_a_non_git_directory(self): + """Organization evidence never depends on a code revision (codex:65a17212478a56416b1c).""" + outside = tempfile.TemporaryDirectory() + self.addCleanup(outside.cleanup) + code, out, err, _ = self.invoke(['attach', '--connection', 'example', '--host', 'codex', + '--request-stdin', '--repo-path', outside.name], self.attach_spec) + self.assertEqual(code, 0, err) + self.assertIn('"status": "attached"', out) + def test_explicit_unavailable_input_requires_fresh_review_and_never_reads_provider(self): from code_mower import audit_labeler_lib spec = {key: value for key, value in self.attach_spec.items() if key != 'packet'} @@ -196,7 +205,7 @@ def setUp(self): self.head = self.manifest.commit self.comments = [] - def invoke(self, args, spec=None, *, actor='controller'): + def invoke(self, args, spec=None, *, actor='controller', remote_head=None): def github(method, path, **kwargs): if path == '/user': return {'login': actor} @@ -212,23 +221,63 @@ def publish(repo, pr, body, **kwargs): stack.enter_context(mock.patch.object(command, '_github', return_value='test-authorization')) stack.enter_context(mock.patch.object(command, '_gh_request', side_effect=github)) stack.enter_context(mock.patch.object(command, 'post_pr_comment', side_effect=publish)) - stack.enter_context(mock.patch.object(command, 'fetch_pull_request', return_value={'head': {'sha': self.head}})) + stack.enter_context(mock.patch.object(command, 'fetch_pull_request', + return_value={'head': {'sha': remote_head if remote_head is not None else self.head}})) stack.enter_context(mock.patch.object(command, 'fetch_issue_comments', return_value=self.comments)) stack.enter_context(mock.patch.object(command.sys, 'stdin', SimpleNamespace(buffer=io.BytesIO(json.dumps(spec).encode())))) code = work_orders.context_main(args) return code, stdout.getvalue(), stderr.getvalue() - def attach(self): + def _attach_spec(self, pr=7): packet_spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', 'query': 'parse_config', 'source': 'impact', 'policy': GRAPH_POLICY} result = context_packets.fetch(self.store, 'local-graph', packet_spec, revision=self.head) - spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'pr': 7, 'policy': GRAPH_POLICY, + return {'repository': 'owner/repo', 'work_item': 'WORK-1', 'pr': pr, 'policy': GRAPH_POLICY, 'packet': result['packet_handle']} + + def attach(self, *, repo_path=None, remote_head=None): + spec = self._attach_spec() code, out, err = self.invoke( - ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin'], spec) + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(repo_path if repo_path is not None else self.repository)], + spec, remote_head=remote_head) self.assertEqual(code, 0, err) return json.loads(out)['revision'] + def test_attachment_succeeds_when_the_checkout_matches_the_pr_head(self): + revision = self.attach() + self.assertTrue(revision) + self.assertEqual(len(self.comments), 1) + + def test_attachment_refuses_a_checkout_that_has_moved_before_publication(self): + """Checkout B must not enable a binding for a PR still reporting head A.""" + spec = self._attach_spec() + graph_fixtures.git(self.repository, 'commit', '--allow-empty', '-q', '-m', 'checkout moved to B') + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(self.repository)], spec) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + + def test_attachment_refuses_a_non_git_directory_before_publication(self): + spec = self._attach_spec() + outside = self.repository.parent / 'not-a-checkout' + outside.mkdir() + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(outside)], spec) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packet_even_when_the_checkout_matches(self): + """Packet A versus PR B is refused even though the consuming checkout is A.""" + spec = self._attach_spec() + code, out, err = self.invoke( + ['attach', '--connection', 'local-graph', '--host', 'codex', '--request-stdin', + '--repo-path', str(self.repository)], spec, remote_head='b' * 40) + self.assertEqual((code, out), (1, '')) + self.assertEqual(self.comments, []) + def test_replay_succeeds_from_the_actual_consuming_checkout(self): revision = self.attach() code, out, err = self.invoke(['deliver', '--revision', revision, '--recipient', 'codex:builder', diff --git a/tests/test_context_delivery.py b/tests/test_context_delivery.py index c17da5e8..38bed5c6 100644 --- a/tests/test_context_delivery.py +++ b/tests/test_context_delivery.py @@ -204,7 +204,7 @@ def setUp(self): result = fetch(self.store, 'local-graph', spec, revision=self.head) self.current = attach(self.store, 'local-graph', result['packet_handle'], self.policy, ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=self.head, - publish=lambda metadata: None) + publish=lambda metadata: None, consuming_revision=self.head) def delivery(self, **kwargs): return deliver(self.store, self.current['revision'], repository='owner/repo', pr=42, head=self.head, @@ -224,3 +224,67 @@ def test_replay_never_silently_substitutes_the_attachment_head(self): self.delivery() with self.assertRaises(ContextError): self.delivery(consuming_revision=None) + + +@unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') +class GraphAttachmentBindingTests(unittest.TestCase): + """A fresh attachment must bind repository evidence to the actual consuming + checkout, never merely to the PR head a caller happens to report + (codex:65a17212478a56416b1c).""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + root = Path(tmp.name).resolve() + self.repository = graph_fixtures.make_repository(root) + private = root / 'private' + private.mkdir(mode=0o700) + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=private, + ) + self.store = ContextStore(private, vault=MemoryVault()) + self.recipients = [f'{host}:{role}' for host in ('claude', 'codex', 'devin') + for role in ('orchestrator', 'builder', 'reviewer')] + graph_connection.connect(self.store, 'local-graph', { + 'repository_root': str(self.repository), 'repositories': ['owner/repo'], + 'recipients': self.recipients, + }) + self.head = self.manifest.commit # checkout A + self.other = 'b' * 40 # a different checkout, B + self.policy = {'schema': 'code_mower.contextPolicy.v1', 'connection': 'local-graph', + 'policy_version': 'v1', 'required': True} + spec = {'repository': 'owner/repo', 'work_item': 'WORK-1', 'recipient': 'codex:orchestrator', + 'query': 'parse_config', 'source': 'impact', 'policy': self.policy} + self.handle = fetch(self.store, 'local-graph', spec, revision=self.head)['packet_handle'] + self.published = [] + + def attach(self, *, head, **kwargs): + return attach(self.store, 'local-graph', self.handle, self.policy, + ContextRequest('owner/repo', 'WORK-1', 'codex:orchestrator'), pr=42, head=head, + publish=self.published.append, **kwargs) + + def test_attachment_succeeds_when_packet_pr_and_consumer_all_match(self): + current = self.attach(head=self.head, consuming_revision=self.head) + self.assertEqual(current['head'], self.head) + self.assertEqual(len(self.published), 1) + + def test_attachment_refuses_a_consumer_the_checkout_is_not_actually_on(self): + """Checkout B must not enable a binding authorized for checkout A.""" + with self.assertRaises(ContextError): + self.attach(head=self.head, consuming_revision=self.other) + self.assertEqual(self.published, []) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packet_even_when_the_consumer_matches(self): + """Packet A versus PR B is refused even though the consumer is A.""" + with self.assertRaises(ContextError): + self.attach(head=self.other, consuming_revision=self.head) + self.assertEqual(self.published, []) + + def test_attachment_never_silently_substitutes_the_pr_head_for_an_unknown_consumer(self): + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + with self.assertRaises(ContextError): + self.attach(head=self.head) + with self.assertRaises(ContextError): + self.attach(head=self.head, consuming_revision=None) + self.assertEqual(self.published, []) diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index 8f8609be..aa7081a2 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -396,27 +396,46 @@ def test_an_optional_wide_answer_is_delivered_rather_than_degraded(self) -> None len(self.load(saved["packet"], "claude:builder").private_payload()["documents"]), 5, ) - def _attach(self, handle: str, head: str): + def _attach(self, handle: str, head: str, *, consuming_revision=None): return context_delivery.reserve_attachment( self.store, "local-graph", handle, POLICY, ContextRequest("owner/repo", "WORK-1", "codex:orchestrator"), - pr=1, head=head, + pr=1, head=head, consuming_revision=consuming_revision, ) def test_attachment_refuses_a_packet_whose_graph_was_rebuilt(self) -> None: handle = self.fetch()["packet_handle"] self.publish() with self.assertRaises(ContextError): - self._attach(handle, self.manifest.commit) + self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) - def test_attachment_binds_the_pull_requests_head_not_the_checkouts(self) -> None: - """A PR at another commit cannot carry this commit's graph evidence.""" + def test_attachment_succeeds_when_packet_pr_and_consumer_all_match(self) -> None: + handle = self.fetch()["packet_handle"] + # The head the graph *is* for, and the checkout it is actually on, both attach. + self.assertEqual( + self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit)["head"], + self.manifest.commit, + ) + + def test_attachment_refuses_a_pull_request_head_that_differs_from_the_packets_revision(self) -> None: + """Packet A versus PR B is refused even though the consumer is A.""" handle = self.fetch()["packet_handle"] - # The head the graph *is* for attaches. - self.assertEqual(self._attach(handle, self.manifest.commit)["head"], self.manifest.commit) consuming = self._second_commit_the_checkout_is_not_on() with self.assertRaises(ContextError): - self._attach(handle, consuming) + self._attach(handle, consuming, consuming_revision=self.manifest.commit) + + def test_attachment_refuses_a_consumer_the_checkout_is_not_actually_on(self) -> None: + """A checkout at another commit must not enable a binding for this PR head.""" + handle = self.fetch()["packet_handle"] + consuming = self._second_commit_the_checkout_is_not_on() + with self.assertRaises(ContextError): + self._attach(handle, self.manifest.commit, consuming_revision=consuming) + + def test_attachment_never_substitutes_the_pr_head_for_an_unknown_consumer(self) -> None: + """A caller that cannot name its consuming revision (e.g. non-Git) fails closed.""" + handle = self.fetch()["packet_handle"] + with self.assertRaises(ContextError): + self._attach(handle, self.manifest.commit) @unittest.skipUnless(os.name == "posix", "private context needs POSIX protections") diff --git a/tests/test_context_guided.py b/tests/test_context_guided.py index d2ec5162..2a76069f 100644 --- a/tests/test_context_guided.py +++ b/tests/test_context_guided.py @@ -645,6 +645,43 @@ def test_unpublished_repository_delivery_still_binds_the_consuming_checkout(self self.associations, self.store, moved, repo_path=self.repository, backend=None, ) + def test_fresh_attachment_refuses_a_moved_checkout_before_any_publication(self): + """A fresh reservation must bind the actual consumer, not merely the + remote PR head (codex:65a17212478a56416b1c).""" + self._advance_the_checkout() + with self.assertRaises(ContextError): + self.attach() + self.assertEqual(self.comments, []) + pending = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(pending["attachment_state"], "pending") + self.assertNotEqual(pending["context_state"], "ready") + + def test_pending_retry_revalidates_the_actual_consumer_before_publication(self): + """A resumed pending/uncertain attachment must revalidate the current + consumer rather than replay a stale reservation, while preserving the + saved intent and its failure diagnostics (codex:65a17212478a56416b1c).""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()), + ): + with self.assertRaises(KeyboardInterrupt): + context_guided.attach_session( + self.associations, self.store, self.record, repo_path=self.repository, + pr=1, backend=None, + ) + pending = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(pending["attachment_state"], "pending") + revision = pending["revision"] + self._advance_the_checkout() + with self.assertRaises(ContextError): + self.attach() + after = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(after["attachment_state"], "pending") + self.assertEqual(after["revision"], revision) + self.assertNotEqual(after["context_state"], "ready") + self.assertEqual(self.comments, []) + if __name__ == "__main__": unittest.main() From 9d85ad1b76f99f339c28379e0bf9baec1e5aadff Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 23:27:08 -0700 Subject: [PATCH 27/33] Complete pending packet cleanup before reconnect (codex:57555c67a42d8aeba514) disconnect() marks the connection disconnected before attempting packet cleanup, and only reports failure as needs_attention. connect() then let a reconnect from any disconnected local graph connection write verified state straight back, regardless of whether that cleanup ever actually ran, so a surviving packet and its delivery/attachment bindings could become authorized again once the graph and approved scope matched. Reconnect now retries packet cleanup under the same connection lock as an idempotent step before writing verified state, even when the preceding disconnect reported the cleanup complete. A cleanup failure during reconnect propagates a ContextError and leaves the connection disconnected, so it stays unable to authorize or load any surviving packet. --- src/code_mower/context_graph_connection.py | 11 ++++++ tests/test_context_graph_connection.py | 41 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/src/code_mower/context_graph_connection.py b/src/code_mower/context_graph_connection.py index 084b6c8e..8f50af15 100644 --- a/src/code_mower/context_graph_connection.py +++ b/src/code_mower/context_graph_connection.py @@ -117,6 +117,17 @@ def connect(store: ContextStore, name: str, spec: Any) -> dict[str, Any]: raise ContextError("this connection name already names a different context provider") if old is not None and saved_state(old, name)["state"] != "disconnected": raise ContextError("connection already exists; disconnect before changing its checkout or scope") + if old is not None: + # A reconnect retries packet cleanup under this same lock before + # authorizing anything again, even when the disconnect that + # preceded it already reported the cleanup complete: a packet or + # delivery binding that survived that attempt must not become + # authorized again just because the graph and scope are unchanged. + try: + from .context_packets import purge_connection + purge_connection(locked) + except Exception as exc: + raise ContextError("pending packet cleanup failed; reconnect refused") from exc state = {"schema": GRAPH_SCHEMA, "connection": name, "provider": PROVIDER, "kind": CONNECTION_KIND, "state": "verified", "repository_root": str(root), "repositories": spec["repositories"], "recipients": spec["recipients"]} diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index aa7081a2..81284ab0 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -256,6 +256,47 @@ def test_disconnecting_disables_the_connection_and_drops_its_packets(self) -> No with self.assertRaises(ContextError): self.load(handle, "claude:builder") + def _reconnect(self, **overrides) -> dict: + spec = { + "repository_root": str(self.repository), + "repositories": ["owner/repo"], + "recipients": RECIPIENTS, + } + spec.update(overrides) + return connection.connect(self.store, "local-graph", spec) + + def test_reconnect_completes_pending_packet_cleanup_that_disconnect_could_not(self) -> None: + """A cleanup failure on disconnect must not let reconnect skip it. + + Reproduces codex:57555c67a42d8aeba514: previously ``connect`` wrote + ``verified`` state straight back once a disconnected connection was + found, regardless of whether the packets it once authorized were ever + actually purged. With the same graph and approved scope, that let a + surviving packet and its delivery/attachment binding become authorized + again. + """ + handle = self.fetch()["packet_handle"] + binding = self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) + with patch("code_mower.context_packets.purge_connection", side_effect=RuntimeError("boom")): + summary = connection.disconnect(self.store, "local-graph") + self.assertEqual((summary["status"], summary["packet_cleanup"]), ("disconnected", "needs_attention")) + with self.assertRaises(ContextError): + self._reconnect() + # Cleanup is still pending: the connection stays disconnected, and + # neither the surviving packet nor its attachment binding is usable. + with self.store.locked("local-graph") as locked: + self.assertEqual(connection.saved_state(locked.read(), "local-graph")["state"], "disconnected") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + with self.assertRaises(ContextError): + context_delivery.read_binding(self.store, binding["revision"]) + # Cleanup now succeeds: reconnect completes it and only then verifies. + self.assertEqual(self._reconnect()["status"], "verified") + with self.assertRaises(ContextError): + self.load(handle, "claude:builder") + with self.assertRaises(ContextError): + context_delivery.read_binding(self.store, binding["revision"]) + def test_connection_status_reports_the_graph_without_minting_evidence(self) -> None: report = connection.status(self.store, "local-graph", root=self.private) self.assertEqual(report["provider"], connection.PROVIDER) From c01e1586a1eff3c6f46eb2f2c7ec8376982cde7e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 15 Sep 2026 23:32:16 -0700 Subject: [PATCH 28/33] Correct reconnect cleanup regression coverage (codex:57555c67a42d8aeba514) read_binding() is a private file lookup with no authorization of its own, so its surviving the failed disconnect/reconnect pair is expected evidence that cleanup is pending, not a security check. Publish the reserved attachment before forcing both cleanup failures so the test replays the strongest path, assert the binding file still exists rather than raising on it, and assert that context_delivery.deliver() -- the actual authorization boundary -- fails for that published binding while the connection stays disconnected. --- tests/test_context_graph_connection.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/test_context_graph_connection.py b/tests/test_context_graph_connection.py index 81284ab0..f1805b2e 100644 --- a/tests/test_context_graph_connection.py +++ b/tests/test_context_graph_connection.py @@ -276,26 +276,37 @@ def test_reconnect_completes_pending_packet_cleanup_that_disconnect_could_not(se again. """ handle = self.fetch()["packet_handle"] - binding = self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) + metadata = self._attach(handle, self.manifest.commit, consuming_revision=self.manifest.commit) + current = context_delivery.mark_published(self.store, "local-graph", metadata["revision"]) with patch("code_mower.context_packets.purge_connection", side_effect=RuntimeError("boom")): summary = connection.disconnect(self.store, "local-graph") self.assertEqual((summary["status"], summary["packet_cleanup"]), ("disconnected", "needs_attention")) with self.assertRaises(ContextError): self._reconnect() - # Cleanup is still pending: the connection stays disconnected, and - # neither the surviving packet nor its attachment binding is usable. + # Cleanup is still pending: the connection stays disconnected and the + # surviving packet cannot be loaded. ``read_binding`` is only a + # private file lookup with no authorization of its own, so the + # binding it names is expected to still be there -- that survival is + # exactly the pending cleanup, not a security property by itself. with self.store.locked("local-graph") as locked: self.assertEqual(connection.saved_state(locked.read(), "local-graph")["state"], "disconnected") with self.assertRaises(ContextError): self.load(handle, "claude:builder") + survived = context_delivery.read_binding(self.store, metadata["revision"]) + self.assertTrue(survived["published"]) + # The actual security property: a published binding must not replay + # while the connection that authorized it is disconnected. with self.assertRaises(ContextError): - context_delivery.read_binding(self.store, binding["revision"]) + context_delivery.deliver( + self.store, metadata["revision"], repository="owner/repo", pr=1, head=self.manifest.commit, + recipient="claude:reviewer", current=current, consuming_revision=self.manifest.commit, + ) # Cleanup now succeeds: reconnect completes it and only then verifies. self.assertEqual(self._reconnect()["status"], "verified") with self.assertRaises(ContextError): self.load(handle, "claude:builder") with self.assertRaises(ContextError): - context_delivery.read_binding(self.store, binding["revision"]) + context_delivery.read_binding(self.store, metadata["revision"]) def test_connection_status_reports_the_graph_without_minting_evidence(self) -> None: report = connection.status(self.store, "local-graph", root=self.private) From de68dea3aac18ef13c208210cc26002cebed8efd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 00:22:07 -0700 Subject: [PATCH 29/33] Preserve Graphify call-site edge identity (codex:90851736427e7bca1693) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GraphEdge discarded the provider edge's source_file/source_location, so run_query treated call-site records that share endpoints, relation, kind and confidence as duplicates and silently dropped all but one -- possibly reporting a complete answer that was not. GraphEdge now retains the call site's path and optional line, holds it to the same citation rules as a node's, and folds it into one canonical identity used for both traversal dedup and the depth-frontier already-reported check. load_graph() and _grouped() now sort by that same full identity instead of the incomplete (kind, target, source) key, so a harmless permutation of the provider's own node or edge order can no longer change which relationships a tight budget reports. A resolvable call site is carried into packet citations under its own "call site: " title, alongside both endpoint citations; an unresolved one is dropped and reported via the existing provider_warning rather than presented as verified. Also fixes _location() accepting Unicode digits (e.g. "L²") that str.isdigit() allows but int() cannot parse, which escaped the bounded ContextError refusal as an unhandled ValueError. Co-Authored-By: Claude Sonnet 5 --- docs/context-graph-queries.md | 81 ++++-- src/code_mower/context_graph_query.py | 152 ++++++++--- tests/test_context_graph_query.py | 373 +++++++++++++++++++++++++- 3 files changed, 544 insertions(+), 62 deletions(-) diff --git a/docs/context-graph-queries.md b/docs/context-graph-queries.md index 6a2e2802..a8f2b801 100644 --- a/docs/context-graph-queries.md +++ b/docs/context-graph-queries.md @@ -59,6 +59,24 @@ unchanged, so reading the edge record reads the provider's semantics. Permuting the node list cannot reverse a relationship, because nothing about the endpoint order is derived from node order. +An edge's own `source_file` and `source_location` are that call site — the file +and, where the extractor recorded one, the line the relationship was written +at. This adapter retains both, holds them to the same citation rules as a +node's location, and carries a resolvable one into the packet, so two call +sites of one relationship are two distinguishable pieces of evidence rather +than one relationship with the second one's location discarded. + +**Upstream limitation.** The pinned release's `--no-cluster` branch itself +calls `build.dedupe_edges`, keyed by `(source, target, relation)`, before +`graph.json` is written, so two call sites the extractor found while walking +the source may already be down to one record by the time this module reads the +artifact. What this adapter guarantees is narrower and still real: every edge +record the artifact actually carries — including two that differ only by call +site, which the supported directed node-link input carries the same way — is +preserved as a distinct relationship rather than collapsed a second time on the +way to a packet. It does not establish exhaustive live-provider call-site +coverage upstream of that dedupe step. + ### The other document: a NetworkX node-link export A generation built through the clustered path instead carries @@ -99,11 +117,17 @@ same for both documents: `confidence` must be uppercase `EXTRACTED`/`INFERRED`/`AMBIGUOUS`. Lowercase is the *packet* vocabulary, and a graph using it was not written by the pinned provider. -- Its locations. `source_location` is `L` or empty; anything else is a - location this module could not check against the bound commit, so it refuses - rather than traversing past it. One line per node, never a span: neither - document records an extent, and claiming one would be this adapter inventing - it. +- Its locations, on a node's `source_location` and an edge's own. Each is + `L` or empty; anything else — including a non-ASCII digit shape such as + `L²`, which reads as a digit but is not one `int()` can parse — is a location + this module could not check against the bound commit, so it refuses rather + than traversing past it or letting the shape escape as an unhandled error. + One line per node or call site, never a span: neither document records an + extent, and claiming one would be this adapter inventing it. A citable path — + a node's `source_file` or an edge's own — is held to the same repository- + scope rules either way: absolute, parent-traversing, and indexer-private + paths are refused, and a record whose path fails that check is not + traversable at all. Three things are deliberately *not* refusals, because a real generation carries them and rejecting them would reject every ordinary one: @@ -164,9 +188,12 @@ product constraint is that default traversals in the evaluated provider returned Each traversal is symbol-first: a target resolves to the symbols carrying that name, and only a target that names no symbol at all is read as a path. Each is -breadth-first over adjacency sorted by `(kind, target, source)`, so one -generation and one question produce one answer, every time, and a budget cut -removes the furthest relationships rather than arbitrary ones. +breadth-first over adjacency sorted by the full relationship identity described +below — source, target, the provider's relation word, its normalized kind and +confidence, and the call site's path and line — so one generation and one +question produce one answer, every time; permuting the provider's own node or +edge order changes nothing about it; and a budget cut removes the furthest +relationships rather than arbitrary ones. A target resolves in three ordered tiers, and a later tier is consulted only when every earlier one is empty: @@ -216,16 +243,21 @@ hops away, which the graph does not carry. What bounds the walk and what bounds the answer are two different things. A node is stepped through once, which is what keeps a traversal linear and terminating. A *relationship* is reported once per distinct provider edge record — its two -endpoints and its own wording together — including when both endpoints have -already been seen. So if `a` calls `b` and `b` calls `a`, both directions are -reported; relationships among the definitions a path target selects as seeds are -reported rather than dropped for having no unseen endpoint; a walk that -reconverges keeps both edges into the node it reached twice; and a self-loop -reached from both sides of a `symbol` neighbourhood, or an edge the provider -recorded twice, is one relationship. Nothing about the bound changes: a -relationship that does not fit the node budget still sets `truncated` and raises -`provider_has_more`, and the depth limit still applies and still reports what it -stopped. +endpoints, its own wording, its normalized confidence, and its own call site +(path and, where the provider recorded one, line) together — including when +both endpoints have already been seen. So if `a` calls `b` and `b` calls `a`, +both directions are reported; relationships among the definitions a path target +selects as seeds are reported rather than dropped for having no unseen +endpoint; a walk that reconverges keeps both edges into the node it reached +twice; a self-loop reached from both sides of a `symbol` neighbourhood, or an +edge the provider recorded twice at the same call site, is one relationship; +and the same relationship recorded at two different call sites — two lines, or +the same line in two different files — is two, because the call site is part +of the identity rather than a detail dropped on the way to reporting it. +Nothing about the bound changes: a relationship that does not fit the node +budget still sets `truncated` and raises `provider_has_more`, and the depth +limit still applies and still reports what it stopped, including when what it +stopped on differs from an already-reported relationship only by call site. ## Citations are validated against the bound commit @@ -247,7 +279,8 @@ drop is reported as `provider_warning`, and a relationship left with no citation at all is reported as `document_limit`. The scope rules are `context_graph`'s, applied twice: at parse time, so a node -that could never be cited is not traversable either, and again at citation time. +or an edge's own call site that could never be cited is not traversable either, +and again at citation time. ## A dropped relationship is two different facts @@ -297,6 +330,16 @@ Confidence maps the provider's own qualification onto the contract's vocabulary: the uncertainty at the packet level and not only per document. A target name that matches more than one definition does the same. +A document's citations are both endpoints of the relationship it states plus, +where the provider recorded a call-site location and it resolves against the +bound commit, the edge's own — titled `call site: ` so a recipient +can tell it apart from either endpoint. A relationship whose call site carries +no location leaves the endpoint citations exactly as before; one whose stated +call site the bound commit does not carry — an untracked path, or a line past +the end of the file — is never presented as verified evidence: the citation is +dropped and the packet still raises `provider_warning`, the same signal an +unresolved endpoint raises. + Document text is metadata about relationships — names, paths, relationship kinds, hop counts — and never indexed content. Everything in it is already in the citations beside it, so the prose adds no claim a recipient cannot check. diff --git a/src/code_mower/context_graph_query.py b/src/code_mower/context_graph_query.py index 75dbc265..ab44a566 100644 --- a/src/code_mower/context_graph_query.py +++ b/src/code_mower/context_graph_query.py @@ -150,10 +150,12 @@ #: this is the whole of the difference. GRAPH_CONFIDENCES = {"EXTRACTED": "extracted", "INFERRED": "inferred", "AMBIGUOUS": "ambiguous"} -#: A node's location in the pinned export is ``source_location``, a string of -#: the form ``L`` written by the extractor's ``add_node``/``add_edge``. -#: Cross-file stubs carry ``""`` -- a real node with no location, which this -#: module keeps traversable and refuses to cite. +#: A node's or an edge's location in the pinned export is ``source_location``, +#: a string of the form ``L`` written by the extractor's +#: ``add_node``/``add_edge``. Cross-file stubs carry ``""`` -- a real node with +#: no location, which this module keeps traversable and refuses to cite -- and +#: an edge's own ``source_location`` is optional the same way: a call site with +#: no recorded line stays traversable and uncitable rather than refused. _LOCATION_MAX_LINE = 10_000_000 #: The pinned extractor's relation vocabulary, normalized onto the @@ -305,6 +307,21 @@ def _callable_base(label: str) -> str: return "" +def _location_citation(path: str, line: int | None) -> str | None: + """A path and an optional line as a citation, or ``None`` with no path. + + Shared by ``GraphNode`` and ``GraphEdge``: the pinned export records a + single line per node or call site, not a span, so a located citation + states one line. Claiming a span the provider never stated would be this + adapter inventing the extent of a definition or a call. + """ + if not path: + return None + if line is None: + return path + return f"{path}#L{line}" + + @dataclass(frozen=True) class GraphNode: """One node of the pinned export, narrowed and held to the citation rules. @@ -326,17 +343,8 @@ class GraphNode: @property def citation(self) -> str | None: - """The node's location as a citation, or ``None`` if it has no location. - - The pinned export records a single line per node, not a span, so a - located node cites one line. Claiming a span the provider never stated - would be this adapter inventing the extent of a definition. - """ - if not self.path: - return None - if self.line is None: - return self.path - return f"{self.path}#L{self.line}" + """The node's location as a citation, or ``None`` if it has no location.""" + return _location_citation(self.path, self.line) @dataclass(frozen=True) @@ -347,6 +355,14 @@ class GraphEdge: normalized onto ``GRAPH_RELATIONS`` for filtering. A packet sentence states ``relation``, so a recipient reads the provider's claim and not this module's grouping of it. + + ``path`` and ``line`` are the provider's own ``source_file`` and + ``source_location`` for this call site, held to the same citation rules as + a node's. Two edges that agree on every other field but were written at + different call sites are two distinct relationships, not one repeated: + without the call site, ``run_query`` had no way to tell "the same claim, + twice" from "the same claim, from two different places in the code" and + silently kept only one. """ source: str @@ -354,6 +370,39 @@ class GraphEdge: relation: str kind: str evidence: str + path: str + line: int | None + + @property + def citation(self) -> str | None: + """The call site's location as a citation, or ``None`` if it has none.""" + return _location_citation(self.path, self.line) + + +#: Sorts below every real provider line, which is always ``>= 1``, so a call +#: site with no line still has a total order against one that has it -- and +#: does so without giving ``line`` a type that mixes ``None`` into a sort key. +_NO_LINE = -1 + + +def _edge_identity(edge: GraphEdge) -> tuple[str, str, str, str, str, str, int]: + """The one identity a retained edge is deduplicated and ordered by. + + Source, target, the provider's own relation word, the normalized kind, + the normalized confidence, and the call site's path and line, together. + Two call-site records that agree on everything else but were written at + different lines -- or in different files -- are two relationships, and + dropping either the path or the line back onto ``None`` is how a reader + would silently collapse them back into one. This is the single identity + ``run_query`` deduplicates and reports truncation against, and the single + key ``load_graph`` and ``_grouped`` sort by, so a permutation of the + provider's node or edge order can never change what a bounded traversal + reports. + """ + return ( + edge.source, edge.target, edge.relation, edge.kind, edge.evidence, + edge.path, edge.line if edge.line is not None else _NO_LINE, + ) @dataclass(frozen=True) @@ -470,9 +519,15 @@ def _location(value: Any) -> int | None: if value is None or value == "": return None text = _text(value, maximum=32) - if not text.startswith("L") or not text[1:].isdigit(): + suffix = text[1:] + # ``str.isdigit()`` accepts Unicode digits ``int()`` cannot parse (a + # superscript ``\u00b2`` is a digit and not a decimal), which would let a + # value like ``"L\u00b2"`` escape this refusal as an unhandled + # ``ValueError`` instead of the bounded ``ContextError`` every other + # unsupported location shape raises. Requiring ASCII first closes that. + if not text.startswith("L") or not suffix.isascii() or not suffix.isdigit(): raise ContextError("local graph source location is not a supported provider location") - line = int(text[1:]) + line = int(suffix) if not 1 <= line <= _LOCATION_MAX_LINE: raise ContextError("local graph line number is out of range") return line @@ -625,16 +680,21 @@ def _edge( target = _text(record["target"], maximum=512) endpoints = ((source, source in nodes), (target, target in nodes)) if all(present for _, present in endpoints): - return _ParsedEdge( - edge=GraphEdge( - source=source, - target=target, - relation=relation, - kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), - evidence=GRAPH_CONFIDENCES[confidence], - ), - incomplete=(), + built = GraphEdge( + source=source, + target=target, + relation=relation, + kind=GRAPH_RELATIONS.get(relation, OTHER_RELATION), + evidence=GRAPH_CONFIDENCES[confidence], + path=_maybe_text(record["source_file"], maximum=1024), + line=_location(record.get("source_location")), ) + # Held to the same citation rules as a node's, at parse time: a call + # site this adapter could never point at inside the indexed checkout + # must not stay traversable as though it were verified evidence. + if built.citation is not None: + parse_graph_citation(built.citation) + return _ParsedEdge(edge=built, incomplete=()) if all(present or endpoint in excluded for endpoint, present in endpoints): # Every absent end was a node the document declared and this module # deliberately does not query. A stated scope, not missing evidence. @@ -668,7 +728,7 @@ def _grouped(edges: Iterable[GraphEdge], *, by: str) -> dict[str, tuple[GraphEdg for edge in edges: buckets.setdefault(getattr(edge, by), []).append(edge) return { - key: tuple(sorted(group, key=lambda edge: (edge.kind, edge.target, edge.source))) + key: tuple(sorted(group, key=_edge_identity)) for key, group in buckets.items() } @@ -747,7 +807,7 @@ def load_graph(payload: Mapping[str, Any], *, generation: str, commit: str) -> C parsed_edges = [_edge(value, nodes, frozen) for value in raw_edges] edges = tuple(sorted( (item.edge for item in parsed_edges if item.edge is not None), - key=lambda edge: (edge.kind, edge.source, edge.target), + key=_edge_identity, )) # Attributed to the surviving endpoint rather than counted: a traversal that # never reaches one of these nodes is not answering over missing evidence @@ -928,7 +988,7 @@ def run_query( # repository is not a hole in *this* answer, and marking every query partial # because of one would make the flag mean nothing. incomplete = any(node.id in graph.incomplete for node in seeds) - reported: set[tuple[str, str, str, str, str]] = set() + reported: set[tuple[str, str, str, str, str, str, int]] = set() relations: list[Relation] = [] over_budget = False frontier: list[tuple[GraphNode, GraphNode, int]] = [(node, node, 0) for node in seeds] @@ -959,17 +1019,18 @@ def run_query( # boundary has already been expanded by the time the first # boundary node is popped. ``reported`` is final here. if not beyond_depth and any( - (edge.source, edge.target, edge.relation, edge.kind, edge.evidence) not in reported + _edge_identity(edge) not in reported for edge, _ in _neighbours(graph, node.id, direction, kinds) ): beyond_depth = True continue for edge, other_id in _neighbours(graph, node.id, direction, kinds): - # The provider's own record, endpoints and wording together: two - # parallel edges that say different things about the same pair are - # two relationships, a self-loop reached from both sides is one, - # and a byte-identical duplicate record is one. - identity = (edge.source, edge.target, edge.relation, edge.kind, edge.evidence) + # The provider's own record, endpoints, wording and call site + # together: two parallel edges that say different things about the + # same pair -- or say the same thing from two different call sites + # -- are two relationships, a self-loop reached from both sides is + # one, and a byte-identical duplicate record is one. + identity = _edge_identity(edge) if identity in reported: continue if len(relations) >= node_budget: @@ -1196,14 +1257,28 @@ def _documents( continue located += 1 endpoints.setdefault(citation, endpoint) - cited = [(citation, endpoint) for citation, endpoint in endpoints.items() - if validator.validate(citation)] + cited = [(citation, f"{endpoint.kind} {endpoint.name}") + for citation, endpoint in endpoints.items() if validator.validate(citation)] if len(cited) != len(endpoints) or located < 2: # The graph claimed a location the bound commit does not carry. # That is the provider disagreeing with the immutable tree, which # a recipient must be told about even when the relationship keeps # a second citation that does check out. unvalidated = True + # The call site itself, titled apart from either endpoint so a + # recipient can tell where the relationship was written from the two + # places it connects. Missing is not unvalidated -- a call site with no + # location is the provider stating none, exactly like a sourceless + # stub's endpoint citation above -- but a stated one the bound commit + # does not carry is the same disagreement an endpoint's is, and must + # never appear as though it were verified. + call_site = item.via.citation + if call_site is not None: + if validator.validate(call_site): + if call_site not in {source for source, _ in cited}: + cited.append((call_site, f"call site: {item.via.relation}")) + else: + unvalidated = True cited = cited[:MAX_CITATIONS_PER_DOCUMENT] if not cited: dropped = True @@ -1220,8 +1295,7 @@ def _documents( # two-endpoint relationship does not label the endpoint it came # from with the name of the one it reached. "citations": [ - {"source": citation, "title": f"{endpoint.kind} {endpoint.name}"} - for citation, endpoint in cited + {"source": source, "title": title} for source, title in cited ], }) if unvalidated: diff --git a/tests/test_context_graph_query.py b/tests/test_context_graph_query.py index 1d533e81..d1ca98e3 100644 --- a/tests/test_context_graph_query.py +++ b/tests/test_context_graph_query.py @@ -547,6 +547,76 @@ def test_refuses_duplicate_node_identifiers(self) -> None: self.load(document) +class EdgeLocationTests(unittest.TestCase): + """The call site's own path and line, held to the same rules as a node's. + + ``source_file`` is a required provider field and every fixture edge carries + a real one; what varies here is ``source_location``, which is optional the + same way a node's is, and the path itself, which is held to the same + repository-scope rules ``test_refuses_a_node_outside_the_indexed_checkout`` + holds a node's ``source_file`` to. + """ + + def load(self, document: dict) -> query.CodeGraph: + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def touched(self, graph: query.CodeGraph) -> query.GraphEdge: + """The one edge these tests mutate: ``n-load`` calls ``n-config``.""" + return next(item for item in graph.edges + if item.source == "n-load" and item.target == "n-config") + + def test_missing_null_and_empty_locations_stay_traversable_and_uncitable(self) -> None: + for mutate in ( + lambda doc: doc["edges"][0].pop("source_location"), + lambda doc: doc["edges"][0].update(source_location=None), + lambda doc: doc["edges"][0].update(source_location=""), + ): + with self.subTest(mutate=mutate): + document = graph_document() + mutate(document) + edge = self.touched(self.load(document)) + self.assertIsNone(edge.line) + # Still traversable, and its citation names the file alone -- + # never a line the provider did not state. + self.assertEqual(edge.citation, "example_pkg/config.py") + + def test_a_malformed_location_type_is_refused(self) -> None: + for value in ([], {}, ["L1"], 12, True): + with self.subTest(value=value): + document = graph_document() + document["edges"][0]["source_location"] = value + with self.assertRaises(ContextError): + self.load(document) + + def test_non_ascii_digits_are_refused_rather_than_crashing(self) -> None: + """``str.isdigit()`` accepts a superscript two; ``int()`` cannot parse it. + + Before this correction ``L²`` escaped as an unhandled + ``ValueError`` instead of the bounded ``ContextError`` every other + unsupported location shape raises. + """ + document = graph_document() + document["edges"][0]["source_location"] = "L²" + with self.assertRaises(ContextError): + self.load(document) + + def test_invalid_line_numbers_are_refused(self) -> None: + for location in ("12", "line 12", "L", "L0", "L-4", "L99999999999"): + with self.subTest(location=location): + document = graph_document() + document["edges"][0]["source_location"] = location + with self.assertRaises(ContextError): + self.load(document) + + def test_an_absolute_traversing_or_private_call_site_path_is_refused(self) -> None: + for path in ("/etc/passwd", "../sibling/config.py", ".git/config", ".graphify/nodes.bin"): + with self.subTest(path=path): + document = graph_document() + document["edges"][0]["source_file"] = path + with self.assertRaises(ContextError): + self.load(document) + + class NodeLinkFormatTests(unittest.TestCase): """The other document that can appear under ``graph.json``, read on its own terms. @@ -564,6 +634,10 @@ def test_reads_a_directed_node_link_export(self) -> None: graph = self.load(node_link_document()) self.assertEqual(len(graph.nodes), 5) self.assertEqual(len(graph.edges), 4) + # The supported node-link document carries the same edge call sites as + # the raw extraction, read by the same ``_edge`` logic. + contains = next(item for item in graph.edges if item.relation == "contains") + self.assertEqual((contains.path, contains.line), ("example_pkg/config.py", 12)) def test_reads_the_renamed_edges_key_of_a_node_link_export(self) -> None: """NetworkX renamed ``links`` to ``edges``; the pinned validator takes either. @@ -821,6 +895,182 @@ def test_depth_still_bounds_a_walk_that_retains_relationships(self) -> None: self.assertEqual(self.stated(result), {("n-b", "calls", "n-a")}) +class CallSiteIdentityTests(unittest.TestCase): + """Two calls to the same relationship, recorded at two places, are two. + + ``edge()`` gives every edge the same call site unless a test overrides it, + so these override it directly. This is the finding's own case: a provider + that records more than one call site for one relationship, and a reader + that dropped the call site from the identity reported one when the + document stated several. + """ + + def load(self, *nodes, edges=()) -> query.CodeGraph: + document = graph_document(nodes=list(nodes), edges=list(edges)) + return query.load_graph(document, generation="a" * 32, commit="b" * 40) + + def sites(self, result: query.QueryResult) -> set: + return {(item.via.path, item.via.line) for item in result.relations} + + def test_same_endpoints_relation_and_confidence_at_different_lines_are_distinct(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/config.py", 9)}, + ) + self.assertFalse(result.truncated) + + def test_the_same_line_in_different_files_is_distinct(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_file="example_pkg/config.py", source_location="L5"), + edge("n-a", "n-b", "calls", source_file="example_pkg/loader.py", source_location="L5"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/loader.py", 5)}, + ) + + def test_an_exact_duplicate_record_still_collapses(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L5"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 1) + self.assertFalse(result.truncated) + self.assertNotIn("provider_has_more", result.omissions) + + def test_a_duplicate_plus_one_distinct_location_yields_two(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertEqual( + self.sites(result), + {("example_pkg/config.py", 5), ("example_pkg/config.py", 9)}, + ) + + def test_different_relations_or_confidences_at_one_call_site_stay_distinct(self) -> None: + """One call site, two different provider claims about it: both survive.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "references", source_location="L5"), + edge("n-a", "n-b", "calls", "INFERRED", source_location="L5"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 3) + + def test_reversed_edge_input_yields_identical_identities_and_order(self) -> None: + """A permutation of the provider's own edge list must not change the answer.""" + edges = [ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + edge("n-a", "n-b", "references", source_location="L5"), + ] + forward = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=edges, + ) + reversed_graph = self.load( + node("n-b", "beta", "example_pkg/loader.py", 40), + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=list(reversed(edges)), + ) + def identity(item: query.GraphEdge) -> tuple: + return (item.source, item.target, item.relation, item.path, item.line) + + self.assertEqual( + [identity(item) for item in forward.edges], + [identity(item) for item in reversed_graph.edges], + ) + first = query.run_query(forward, question="dependency", target="alpha") + second = query.run_query(reversed_graph, question="dependency", target="alpha") + self.assertEqual( + [(item.via.relation, item.via.path, item.via.line) for item in first.relations], + [(item.via.relation, item.via.path, item.via.line) for item in second.relations], + ) + + def test_incoming_outgoing_and_symbol_traversal_use_the_full_identity(self) -> None: + """Two distinct call sites survive whichever direction reaches them.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-a", "n-b", "calls", source_location="L9"), + ], + ) + self.assertEqual(len(query.run_query(graph, question="dependency", target="alpha").relations), 2) + self.assertEqual(len(query.run_query(graph, question="impact", target="beta").relations), 2) + self.assertEqual(len(query.run_query(graph, question="symbol", target="alpha").relations), 2) + + def test_a_self_loop_at_distinct_call_sites_is_two_relationships(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + edges=[ + edge("n-a", "n-a", "calls", source_location="L5"), + edge("n-a", "n-a", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + + def test_a_cycle_at_distinct_call_sites_reports_every_record(self) -> None: + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + edges=[ + edge("n-a", "n-b", "calls", source_location="L5"), + edge("n-b", "n-a", "calls", source_location="L9"), + ], + ) + result = query.run_query(graph, question="symbol", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertFalse(result.truncated) + + def test_relationships_among_path_seeds_stay_one_when_the_call_site_repeats(self) -> None: + """Every endpoint is a seed and the record is not repeated: still one.""" + graph = self.load( + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/config.py", 20), + edges=[edge("n-a", "n-b", "calls", source_location="L5")], + ) + result = query.run_query(graph, question="dependency", target="example_pkg/config.py") + self.assertEqual({item.id for item in result.seeds}, {"n-a", "n-b"}) + self.assertEqual(len(result.relations), 1) + + class DepthBoundaryTests(unittest.TestCase): """What the requested depth left behind, said out loud. @@ -918,6 +1168,15 @@ def test_a_boundary_relationship_back_into_the_answer_is_still_an_omission(self) self.assertTrue(result.truncated) self.assertIn("provider_has_more", result.omissions) + def test_a_boundary_relationship_that_differs_only_by_call_site_is_still_an_omission(self) -> None: + """Same endpoints and relation, two recorded call sites: two identities.""" + graph = self.chain(tail=[edge("n-c", "n-a", "calls", source_location="L1"), + edge("n-c", "n-a", "calls", source_location="L2")]) + result = query.run_query(graph, question="dependency", target="alpha") + self.assertEqual(len(result.relations), 2) + self.assertTrue(result.truncated) + self.assertIn("provider_has_more", result.omissions) + def test_direction_decides_what_the_boundary_counts(self) -> None: """One graph, two questions: the boundary edge points the wrong way for one.""" graph = self.chain(tail=[edge("n-d", "n-c", "calls")]) @@ -982,6 +1241,109 @@ def test_the_rest_of_the_walk_is_unchanged(self) -> None: self.assertNotIn("provider_has_more", self.context().summary["omissions"]) +def call_site_document(*sites: tuple, relation: str = "calls") -> dict: + """``alpha`` and ``beta``, connected once per call site in ``sites``. + + Each entry is a distinct ``(path, line)`` the same relationship was + recorded at -- the finding's own case, where a provider stating one + relationship at more than one call site had all but one silently dropped. + """ + return graph_document( + nodes=[ + node("n-a", "alpha", "example_pkg/config.py", 12), + node("n-b", "beta", "example_pkg/loader.py", 40), + ], + edges=[ + edge("n-a", "n-b", relation, source_file=path, source_location=f"L{line}") + for path, line in sites + ], + ) + + +class CallSitePacketTests(GraphWorkspace): + """What a recipient reads when the provider records more than one call site.""" + + document = call_site_document(("example_pkg/config.py", 5), ("example_pkg/config.py", 9)) + + def query(self, **overrides) -> query.GraphContext: + arguments = {"question": "dependency", "target": "alpha"} + arguments.update(overrides) + return self.context(**arguments) + + def test_distinct_call_sites_produce_distinguishable_documents_and_citations(self) -> None: + outcome = self.query() + self.assertEqual(outcome.status, query.AVAILABLE) + self.assertEqual(len(outcome.packet["documents"]), 2) + citations = [ + {citation["source"] for citation in item["citations"]} + for item in outcome.packet["documents"] + ] + every_source = {source for group in citations for source in group} + self.assertIn("example_pkg/config.py#L5", every_source) + self.assertIn("example_pkg/config.py#L9", every_source) + # Each document names the call site its own relationship record was + # actually written at, not the other one -- that is what makes them + # distinguishable evidence rather than the same document twice. + self.assertNotEqual(citations[0], citations[1]) + self.assertFalse(outcome.packet["truncated"]) + + def test_reversed_edge_input_produces_identical_packet_documents(self) -> None: + forward = self.query().packet["documents"] + reversed_document = call_site_document( + ("example_pkg/config.py", 9), ("example_pkg/config.py", 5)) + reversed_document["nodes"] = list(reversed(reversed_document["nodes"])) + reversed_document["edges"] = list(reversed(reversed_document["edges"])) + self.publish(reversed_document) + reversed_documents = self.query().packet["documents"] + self.assertEqual( + [item["text"] for item in forward], + [item["text"] for item in reversed_documents], + ) + self.assertEqual( + [{c["source"] for c in item["citations"]} for item in forward], + [{c["source"] for c in item["citations"]} for item in reversed_documents], + ) + + def test_a_real_second_call_site_beyond_the_relationship_budget_sets_provider_has_more(self) -> None: + outcome = self.query(node_budget=1) + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertTrue(outcome.packet["truncated"]) + self.assertIn("provider_has_more", outcome.packet["omissions"]) + + def test_a_document_budget_below_the_call_site_count_sets_document_limit(self) -> None: + outcome = self.query(policy=policy(max_documents=1)) + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertTrue(outcome.packet["truncated"]) + self.assertIn("document_limit", outcome.packet["omissions"]) + + def test_duplicate_only_input_does_not_falsely_truncate(self) -> None: + self.publish(call_site_document( + ("example_pkg/config.py", 5), ("example_pkg/config.py", 5))) + outcome = self.query() + self.assertEqual(len(outcome.packet["documents"]), 1) + self.assertFalse(outcome.packet["truncated"]) + self.assertNotIn("provider_has_more", outcome.packet["omissions"]) + + def test_an_unresolved_call_site_citation_is_omitted_and_warned(self) -> None: + """A stated call site past the end of a tracked file is never verified.""" + self.publish(call_site_document(("example_pkg/config.py", 999))) + outcome = self.query() + self.assertEqual(outcome.status, query.AVAILABLE) + [document] = outcome.packet["documents"] + sources = {citation["source"] for citation in document["citations"]} + self.assertNotIn("example_pkg/config.py#L999", sources) + self.assertEqual(sources, {"example_pkg/config.py#L12", "example_pkg/loader.py#L40"}) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + def test_an_untracked_call_site_path_is_never_cited(self) -> None: + self.publish(call_site_document(("example_pkg/untracked.py", 1))) + outcome = self.query() + [document] = outcome.packet["documents"] + sources = {citation["source"] for citation in document["citations"]} + self.assertNotIn("example_pkg/untracked.py#L1", sources) + self.assertIn("provider_warning", outcome.packet["omissions"]) + + def missing_endpoint_document() -> dict: """The shared fixture, plus a relationship onto an id it never declares. @@ -1336,15 +1698,18 @@ def test_multi_hop_evidence_names_and_cites_the_edge_it_walked(self) -> None: [text] = [item for item in documents if "render" in item] self.assertIn("render calls load", text) self.assertIn("reached from parse_config", text) + # The two endpoints, plus the edge's own call site -- ``edge()``'s own + # location, distinct from both endpoints in this fixture. self.assertEqual( {citation["source"] for citation in documents[text]["citations"]}, - {"example_pkg/report.py#L5", "example_pkg/loader.py#L40"}, + {"example_pkg/report.py#L5", "example_pkg/loader.py#L40", "example_pkg/config.py#L12"}, ) - # Each citation is titled with the node it points at, not with the node - # the relationship happened to reach. + # Each endpoint citation is titled with the node it points at, not with + # the node the relationship happened to reach; the call site is titled + # apart from either. self.assertEqual( {citation["title"] for citation in documents[text]["citations"]}, - {"symbol render", "symbol load"}, + {"symbol render", "symbol load", "call site: calls"}, ) def test_packet_text_carries_no_indexed_content(self) -> None: From a86cef2e27818bbf9a65c04c096e7897e171af4d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 01:13:17 -0700 Subject: [PATCH 30/33] Restore guided attachment recovery (codex:1a4bc7b34687da726908) A fresh or re-minted attachment reservation that fails before publication now rolls the session back to a refreshable prepared/none state instead of stranding a permanently pending intent that every retry reauthorizes and that `prepare --refresh` alone could never clear. A saved pending/uncertain intent for an old PR head is retired before its evidence is ever reauthorized, and stale-binding cleanup is retry-safe across both of its write boundaries, with the targeted binding always identity-checked and a malformed remote head never treated as authoritative movement. --- docs/context-delivery.md | 16 +- src/code_mower/context_delivery.py | 20 ++- src/code_mower/context_guided.py | 122 +++++++++---- tests/test_context_delivery.py | 63 ++++++- tests/test_context_guided.py | 271 ++++++++++++++++++++++++++++- 5 files changed, 452 insertions(+), 40 deletions(-) diff --git a/docs/context-delivery.md b/docs/context-delivery.md index bf3dfe28..0edfd86b 100644 --- a/docs/context-delivery.md +++ b/docs/context-delivery.md @@ -161,7 +161,21 @@ that exact revision from the trusted current comment. If a write returns an uncertain result and the revision is not current, status pauses the workflow; after independently checking the PR, `attach --retry-uncertain` republishes the same revision rather than creating another one. No uncertain or unpublished -binding can deliver evidence. +binding can deliver evidence. A saved uncertain or pending intent is never +cleared just because a later attach was asked for: it may already be the +GitHub-accepted state a lost response only looked like it missed. + +A reservation that fails before publication -- for example because the actual +consuming checkout has moved past the evidence it was prepared from, or the +local graph was rebuilt in the meantime -- rolls the session back to its prior +prepared, unattached state on its own, keeping the failure reason visible in +status. The same session can then explicitly rebuild or rerun `prepare +--refresh` and attach again; it is never left pointing at a saved intent that +every retry would only re-fail. Once the trusted current pull request head has +genuinely moved past a saved intent, that old identity is retired before any +new evidence is authorized, so a checkout that has since moved on cannot block +its own recovery. This retirement, like the rollback above, is safe to retry +after an interruption partway through. The lower-level expert form remains available for scripts that intentionally manage request files and revisions: diff --git a/src/code_mower/context_delivery.py b/src/code_mower/context_delivery.py index 30d1a035..fd7075a9 100644 --- a/src/code_mower/context_delivery.py +++ b/src/code_mower/context_delivery.py @@ -205,6 +205,16 @@ def mark_published(store, name, revision): def _remove_attachment(store, name, handle, revision, *, published): + """Remove one identity-checked binding, idempotent for its own interrupted cleanup. + + The index is written without ``revision`` before the artifact is deleted, + so a crash between those two writes leaves an artifact whose index entry + already omits it. Retrying with the exact same ``handle``/``revision`` + recognizes that state -- the artifact's own binding still names them -- + and finishes deleting the artifact rather than reporting an inconsistent + index. A missing or mismatched identity, or a missing index entry, is + never treated as that same interrupted cleanup and still fails closed. + """ _handle(handle) _handle(revision) with store.locked(name) as locked: @@ -218,12 +228,16 @@ def _remove_attachment(store, name, handle, revision, *, published): index_file.write(index) return binding = _binding(saved) + if binding["handle"] != handle or binding["revision"] != revision: + raise ContextError("context attachment index is inconsistent") if binding["published"] and not published: raise ContextError("published context attachment cannot be abandoned") - if entry is None or revision not in entry.setdefault("deliveries", []): + if entry is None: raise ContextError("context attachment index is inconsistent") - entry["deliveries"].remove(revision) - index_file.write(index) + deliveries = entry.setdefault("deliveries", []) + if revision in deliveries: + deliveries.remove(revision) + index_file.write(index) artifact.delete() diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index f5faa82e..5d323985 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib +import re import uuid from pathlib import Path from typing import Any, Mapping @@ -31,6 +32,7 @@ ATTACH_SCHEMA = "code_mower.contextSessionAttach.v1" +_HEAD_SHA = re.compile(r"[a-f0-9]{40}\Z") def _workflow_key(record: Mapping[str, Any]) -> str: @@ -64,6 +66,12 @@ def _remote_input( head = pull["head"]["sha"] except (KeyError, TypeError): raise ContextError("GitHub did not return a valid pull request head") from None + # Malformed remote data (missing, null, wrong-type, shortened, or + # non-hex) is not authoritative head movement: refuse before any + # comparison or cleanup ever sees it, rather than let it masquerade as a + # moved or matching head. + if not isinstance(head, str) or not _HEAD_SHA.fullmatch(head): + raise ContextError("GitHub did not return a valid pull request head") current = context_review.latest_input( fetch_issue_comments(repository, pr, token=token), authorities=authorities, ) @@ -172,7 +180,13 @@ def _discard_stale_attachment( packet_store: ContextStore, record: Mapping[str, Any], ) -> dict[str, Any]: - """Clear one binding only after the live PR head has moved beyond it.""" + """Clear one binding only after the live PR head has moved beyond it. + + Retiring the packet-store binding and updating the association are two + separate writes; either alone is retry-safe (a missing binding is a + no-op for ``retire_attachment``), so an interruption between them is + resolved by simply calling this again with the still-saved record. + """ retire_attachment( packet_store, record["connection"], record["packet"], record["revision"], ) @@ -185,6 +199,48 @@ def _discard_stale_attachment( "attachment_state": "none", }, ) + + +def _abandon_and_clear( + association_store: ContextStore, + packet_store: ContextStore, + record: Mapping[str, Any], +) -> None: + """Best-effort return of a not-yet-published intent to prepared/none. + + Used only when ``record["revision"]`` is known to have not reached + ``_finish_publication``'s remote publication check, so any stored + delivery binding for it is never published and safe to abandon. Never + used to blanket-clear a pre-existing pending/uncertain retry. + ``record_failure`` may already have advanced the association generation, + so the session is re-read here rather than trusting a stale one; if a + concurrent operation has already moved the saved revision on, cleanup is + skipped rather than overwriting that operation's result. A + ``ContextError`` or storage exception from either step does not prove no + local write occurred, so on failure the saved intent is left in place + instead of being claimed cleared -- the caller's own error still reports + the original failure. + """ + try: + current = context_session.read(association_store, record["session_id"]) + if current is None or current["revision"] != record["revision"]: + return + abandon_attachment( + packet_store, current["connection"], current["packet"], current["revision"], + ) + context_session.update( + association_store, + current["session_id"], + expected_generation=current["generation"], + changes={ + "stage": "prepared", "pr": None, "head": None, "revision": None, + "attachment_state": "none", + }, + ) + except (ContextError, OSError): + pass + + def attach_session( association_store: ContextStore, packet_store: ContextStore, @@ -219,8 +275,13 @@ def attach_session( if record["attachment_state"] == "published": if record["pr"] != pr: raise ContextError("this session is already attached to a different pull request") - binding = read_binding(packet_store, record["revision"]) if record["head"] == head: + # Only the same-head path needs the existing binding read. + # A moved head retires by identity below without ever + # reading it, so an interrupted prior retirement's missing + # binding is not read here as current evidence; a same-head + # missing binding still fails closed. + binding = read_binding(packet_store, record["revision"]) if current != binding["metadata"]: raise ContextError( "the trusted current input changed; inspect the pull request before replacing it" @@ -249,11 +310,20 @@ def attach_session( if record["attachment_state"] in {"pending", "uncertain"}: if record["pr"] != pr: raise ContextError("a saved attachment intent targets a different pull request") - metadata = _reserve_for_record( - association_store, packet_store, record, repo_path=repo_path, backend=backend, - ) - if current == metadata: - if record["head"] == head: + if record["head"] != head: + # Retire the old identity before ever reauthorizing its + # evidence. Reserving first, as below, reauthorizes the + # stale saved head's evidence; a checkout that has since + # moved on could fail that reauthorization before this + # already-safe retirement path ever ran. + record = _discard_stale_attachment( + association_store, packet_store, record, + ) + else: + metadata = _reserve_for_record( + association_store, packet_store, record, repo_path=repo_path, backend=backend, + ) + if current == metadata: mark_published(packet_store, record["connection"], record["revision"]) record = context_session.update( association_store, @@ -265,16 +335,8 @@ def attach_session( }, ) return _report("attached", reused=True, reconciled=True), 0 - record = _discard_stale_attachment( - association_store, packet_store, record, - ) - elif record["attachment_state"] == "uncertain" and not retry_uncertain: - return _report("attachment_uncertain", reused=True), 1 - elif record["head"] != head: - record = _discard_stale_attachment( - association_store, packet_store, record, - ) - elif record["attachment_state"] in {"pending", "uncertain"}: + if record["attachment_state"] == "uncertain" and not retry_uncertain: + return _report("attachment_uncertain", reused=True), 1 return _finish_publication( association_store, packet_store, @@ -294,9 +356,17 @@ def attach_session( "attachment_state": "pending", }, ) - metadata = _reserve_for_record( - association_store, packet_store, record, repo_path=repo_path, backend=backend, - ) + try: + metadata = _reserve_for_record( + association_store, packet_store, record, repo_path=repo_path, backend=backend, + ) + except ContextError: + # This exact intent was just minted in this same call and never + # reached publication, so it is safe to abandon; a pre-existing + # pending/uncertain intent retried above is never cleared this + # way. + _abandon_and_clear(association_store, packet_store, record) + raise return _finish_publication( association_store, packet_store, @@ -318,17 +388,7 @@ def _finish_publication( ) -> tuple[dict[str, Any], int]: head, current = _remote_input(record["repo"], record["pr"], token=token, authorities=authorities) if head != record["head"]: - abandon_attachment( - packet_store, record["connection"], record["packet"], record["revision"], - ) - context_session.update( - association_store, - record["session_id"], - expected_generation=record["generation"], - changes={ - "pr": None, "head": None, "revision": None, "attachment_state": "none", - }, - ) + _abandon_and_clear(association_store, packet_store, record) raise ContextError("pull request head changed before publication; rerun attach") if current == metadata: mark_published(packet_store, record["connection"], record["revision"]) diff --git a/tests/test_context_delivery.py b/tests/test_context_delivery.py index 38bed5c6..9d4a4ee6 100644 --- a/tests/test_context_delivery.py +++ b/tests/test_context_delivery.py @@ -11,9 +11,10 @@ from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_connections import connect, disconnect from code_mower.context_contract import ContextError, ContextRequest, load_packet -from code_mower.context_delivery import (SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, attach, deliver, public_verdict, - read_binding, render_evidence, save_feedback) -from code_mower.context_packets import fetch +from code_mower.context_delivery import (SUPPORTED_HOSTS, SUPPORTED_RECIPIENTS, abandon_attachment, attach, + deliver, public_verdict, read_binding, render_evidence, + retire_attachment, save_feedback) +from code_mower.context_packets import _index, fetch from code_mower.context_store import ContextStore from test_context_connections import MemoryVault from test_context_packets import RetrievalBackend @@ -171,6 +172,62 @@ def test_local_repository_graph_uses_common_renderer_without_oauth_identity(self self.assertNotIn('/example/repository', texts[0]) self.assertNotIn('principal', texts[0]) + def test_retirement_completes_after_an_interrupted_index_write(self): + """Delivery removal writes the index without the revision before + deleting the artifact; a crash between those writes must resolve on + retry rather than report an inconsistent index (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + index_file, index = _index(locked) + entry = next(item for item in index['entries'] if item['handle'] == handle) + entry['deliveries'].remove(revision) + index_file.write(index) + # The artifact is still present; only the index write completed. + retire_attachment(self.store, 'example', handle, revision) + with self.assertRaises(ContextError): + read_binding(self.store, revision) + # Retrying the exact same removal is idempotent. + retire_attachment(self.store, 'example', handle, revision) + + def test_abandon_still_refuses_a_published_binding(self): + current = self.attach() + with self.assertRaises(ContextError): + abandon_attachment(self.store, 'example', self.result['packet_handle'], current['revision']) + self.assertTrue(read_binding(self.store, current['revision'])['published']) + + def test_removal_refuses_a_binding_whose_identity_does_not_match(self): + """The targeted binding is identity checked; a mismatched handle or + revision inside the stored binding must still fail closed even + though the artifact exists (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + artifact = locked.artifact('d-' + revision) + corrupted = {**artifact.read(), 'handle': 'f' * 32} + artifact.write(corrupted) + with self.assertRaises(ContextError): + retire_attachment(self.store, 'example', handle, revision) + with self.assertRaises(ContextError): + abandon_attachment(self.store, 'example', handle, revision) + + def test_removal_refuses_when_the_index_entry_is_missing(self): + """A missing index entry is not the same state an interrupted + cleanup leaves -- that only ever drops the revision from an + existing entry's deliveries -- so it still fails closed rather than + completing as if it were (codex:1a4bc7b34687da726908).""" + current = self.attach() + revision = current['revision'] + handle = self.result['packet_handle'] + with self.store.locked('example') as locked: + index_file, index = _index(locked) + index['entries'] = [item for item in index['entries'] if item['handle'] != handle] + index_file.write(index) + with self.assertRaises(ContextError): + retire_attachment(self.store, 'example', handle, revision) + @unittest.skipUnless(os.name == 'posix', 'private store requires POSIX') class GraphDeliveryBindingTests(unittest.TestCase): diff --git a/tests/test_context_guided.py b/tests/test_context_guided.py index 2a76069f..b31cfbb5 100644 --- a/tests/test_context_guided.py +++ b/tests/test_context_guided.py @@ -6,6 +6,7 @@ import os import tempfile import unittest +import uuid from pathlib import Path from unittest import mock @@ -13,7 +14,8 @@ from code_mower import context_graph_connection as graph_connection from code_mower import context_graph_lifecycle as lifecycle from code_mower.context_contract import ContextError -from code_mower.context_delivery import deliver, read_binding, save_feedback +from code_mower.context_delivery import deliver, read_binding, retire_attachment, save_feedback +from code_mower.context_packets import _index from code_mower.context_review import INPUT_HEADER from code_mower.context_store import ContextStore import test_context_delivery as fixtures @@ -406,6 +408,175 @@ def test_status_closes_pending_and_uncertain_recovery_states(self): for private in (saved["revision"], saved["packet"], "EXAMPLE-1", "owner/repo"): self.assertNotIn(private, encoded) + def test_stale_pending_intent_retires_before_reauthorizing_its_own_evidence(self): + """When the trusted remote head has moved beyond a saved pending + intent, the old identity must be retired before the state machine + ever calls ``reserve_attachment`` again for that stale evidence + (codex:1a4bc7b34687da726908).""" + self.fail_comment = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + self.attach() + pending = context_session.read(self.associations, self.session["id"]) + old_revision = pending["revision"] + old_head = pending["head"] + self.head = "d" * 40 + + real_reserve = context_guided.reserve_attachment + + def spying_reserve(*args, **kwargs): + if kwargs.get("head") == old_head: + raise AssertionError("stale evidence was reauthorized before retirement") + return real_reserve(*args, **kwargs) + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=spying_reserve), + ): + current = context_session.read(self.associations, self.session["id"]) + report, code = context_guided.attach_session( + self.associations, self.fixture.store, current, repo_path=self.root, pr=42, + backend=self.fixture.backend, + ) + self.assertEqual((report["status"], code), ("attached", 0)) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + self.assertNotEqual( + context_session.read(self.associations, self.session["id"])["revision"], old_revision, + ) + + def test_interrupted_association_update_recovers_after_stale_retirement(self): + """A crash between retiring a stale published binding and updating + the session association must be retry-safe: the next call finishes + the same identity-checked cleanup rather than reading the now-missing + binding as current evidence (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + old_revision = published["revision"] + retire_attachment( + self.fixture.store, published["connection"], published["packet"], old_revision, + ) + self.head = "d" * 40 + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + saved = context_session.read(self.associations, self.session["id"]) + self.assertNotEqual(saved["revision"], old_revision) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + self.assertEqual(len(self.comments), 2) + + def test_interrupted_index_write_during_retirement_completes_on_retry(self): + """A crash between the index write and the artifact delete inside + delivery cleanup must complete on the next attach rather than + report an inconsistent index (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + old_revision = published["revision"] + with self.fixture.store.locked(published["connection"]) as locked: + index_file, index = _index(locked) + entry = next(item for item in index["entries"] if item["handle"] == published["packet"]) + entry["deliveries"].remove(old_revision) + index_file.write(index) + self.head = "d" * 40 + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + saved = context_session.read(self.associations, self.session["id"]) + self.assertNotEqual(saved["revision"], old_revision) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, old_revision) + + def test_malformed_remote_head_cannot_retire_or_clear_a_published_binding(self): + """Missing, null, wrong-type, shortened, and non-hex ``head.sha`` + values are not authoritative head movement and must never retire or + clear an existing binding or session (codex:1a4bc7b34687da726908).""" + self.attach() + published = context_session.read(self.associations, self.session["id"]) + missing = object() + for malformed in (missing, None, 123, ["d" * 40], "d" * 39, "d" * 41, "g" * 40, "D" * 40): + with self.subTest(head=malformed): + def _pull(*_a, _sha=malformed, **_k): + return {"head": {}} if _sha is missing else {"head": {"sha": _sha}} + with self.patches()[0], mock.patch.object( + context_guided, "fetch_pull_request", side_effect=_pull, + ), self.patches()[2], self.patches()[3], self.patches()[4]: + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.fixture.store, published, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, published) + self.assertTrue(read_binding(self.fixture.store, published["revision"])["published"]) + + def test_malformed_remote_head_cannot_disturb_a_pending_intent(self): + self.fail_comment = KeyboardInterrupt() + with self.assertRaises(KeyboardInterrupt): + self.attach() + pending = context_session.read(self.associations, self.session["id"]) + with self.patches()[0], mock.patch.object( + context_guided, "fetch_pull_request", side_effect=lambda *_a, **_k: {"head": {"sha": None}}, + ), self.patches()[2], self.patches()[3], self.patches()[4]: + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.fixture.store, pending, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, pending) + + def test_concurrent_reconciliation_is_not_overwritten_by_the_rollback(self): + """If another operation already reconciled the exact freshly minted + intent before the rollback runs, the rollback must not overwrite that + result or claim it cleared anything (codex:1a4bc7b34687da726908).""" + real_record_failure = context_session.record_failure + + def racing_record_failure(store, record, error): + updated = real_record_failure(store, record, error) + # A concurrent operation reconciles the exact same intent first. + return context_session.update( + store, record["session_id"], expected_generation=updated["generation"], + changes={ + "stage": "prepared", "pr": None, "head": None, "revision": None, + "attachment_state": "none", + }, + ) + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_session, "record_failure", side_effect=racing_record_failure), + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + after = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (after["attachment_state"], after["pr"], after["head"], after["revision"]), + ("none", None, None, None), + ) + + def test_rollback_failure_retains_the_pending_intent_rather_than_claiming_success(self): + """If the abandon step itself fails, the saved pending intent is left + in place rather than reported as cleared -- a ``ContextError`` there + does not prove no local write occurred (codex:1a4bc7b34687da726908).""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + mock.patch.object(context_guided, "abandon_attachment", side_effect=ContextError("cleanup failed")), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + pending = context_session.read(self.associations, self.session["id"]) + self.assertEqual(pending["attachment_state"], "pending") + self.assertIsNotNone(pending["revision"]) + def test_provider_free_cross_process_qualification_is_symmetric_for_both_hosts(self): delivered = [] @@ -587,6 +758,14 @@ def _advance_the_checkout(self): graph_fixtures.git(self.repository, "add", ".") graph_fixtures.git(self.repository, "commit", "-q", "-m", "second") + def _advance_without_disturbing_citations(self): + """Move HEAD to a new commit without touching any file the fixture's + synthetic graph cites, so a graph rebuilt at the new commit still + validates the same citations.""" + (self.repository / "NOTES.md").write_text("advance\n", encoding="utf-8") + graph_fixtures.git(self.repository, "add", ".") + graph_fixtures.git(self.repository, "commit", "-q", "-m", "advance") + def test_repository_delivery_matches_the_actual_consuming_checkout(self): report, code = self.attach() self.assertEqual((report["status"], code), ("attached", 0)) @@ -653,9 +832,97 @@ def test_fresh_attachment_refuses_a_moved_checkout_before_any_publication(self): self.attach() self.assertEqual(self.comments, []) pending = context_session.read(self.associations, self.SESSION_ID) - self.assertEqual(pending["attachment_state"], "pending") + self.assertEqual(pending["attachment_state"], "none") self.assertNotEqual(pending["context_state"], "ready") + def test_fresh_attachment_recovers_after_checkout_and_pr_advance(self): + """A fresh reservation that fails before publication must roll back + to a refreshable prepared/none state -- with zero public writes and + no leaked usable binding -- rather than stranding a permanently + pending intent that every retry reauthorizes and that + ``prepare --refresh`` alone could never clear + (codex:1a4bc7b34687da726908).""" + self._advance_without_disturbing_citations() + self.head = lifecycle.resolve_revision(self.repository)[0] + fixed = uuid.UUID(hex="1" * 32) + with mock.patch.object(context_guided.uuid, "uuid4", return_value=fixed): + with self.assertRaises(ContextError): + self.attach() + self.assertEqual(self.comments, []) + with self.assertRaises(ContextError): + read_binding(self.store, fixed.hex) + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + self.assertEqual( + (recovered["pr"], recovered["head"], recovered["revision"]), (None, None, None), + ) + self.assertEqual(recovered["stage"], "prepared") + self.assertNotEqual(recovered["context_state"], "ready") + + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + report, code = context_prepare.prepare( + self.associations, recovered, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + refresh=True, + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + attached = context_session.read(self.associations, self.SESSION_ID) + self.assertIsNotNone(attached["revision"]) + self.assertNotEqual(attached["revision"], fixed.hex) + + def test_same_head_failures_recover_without_stranding_refresh(self): + """A same-head graph generation replacement, or an unresolvable + consuming revision, must roll back exactly like a moved checkout -- + never stranding the session on a permanently pending intent + (codex:1a4bc7b34687da726908).""" + # The graph is rebuilt for the same commit, minting a new generation + # the saved packet was never authorized against. + lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + with self.assertRaises(ContextError): + self.attach() + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + self.assertEqual( + (recovered["pr"], recovered["head"], recovered["revision"]), (None, None, None), + ) + + # The consuming checkout cannot be resolved at all. + outside = self.root / "not-a-checkout" + outside.mkdir() + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + ): + current = context_session.read(self.associations, self.SESSION_ID) + with self.assertRaises(ContextError): + context_guided.attach_session( + self.associations, self.store, current, repo_path=outside, pr=1, backend=None, + ) + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(recovered["attachment_state"], "none") + + # The same session can still explicitly refresh and later attach. + self.manifest = lifecycle.build_graph( + self.repository, pin=graph_fixtures.PIN, + indexer=graph_fixtures.indexer(graph_fixtures.graph_document()), root=self.private, + ) + report, code = context_prepare.prepare( + self.associations, recovered, repo_root=self.repository, context_root=self.private, + packet_store=self.store, query="parse_config", source="impact", builder="codex", + refresh=True, + ) + self.assertEqual((code, report["status"]), (0, "prepared")) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + def test_pending_retry_revalidates_the_actual_consumer_before_publication(self): """A resumed pending/uncertain attachment must revalidate the current consumer rather than replay a stale reservation, while preserving the From bcabc16f60572ae5a11525fc9df405c1d8cd619f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 01:45:29 -0700 Subject: [PATCH 31/33] Make attachment rollback resumable (codex:1a4bc7b34687da726908) --- docs/context-delivery.md | 42 ++++-- src/code_mower/context_guided.py | 43 +++++- src/code_mower/context_session.py | 17 ++- tests/test_context_guided.py | 224 ++++++++++++++++++++++++++++-- tests/test_context_session.py | 39 ++++++ 5 files changed, 336 insertions(+), 29 deletions(-) diff --git a/docs/context-delivery.md b/docs/context-delivery.md index 0edfd86b..a01416c3 100644 --- a/docs/context-delivery.md +++ b/docs/context-delivery.md @@ -165,17 +165,37 @@ binding can deliver evidence. A saved uncertain or pending intent is never cleared just because a later attach was asked for: it may already be the GitHub-accepted state a lost response only looked like it missed. -A reservation that fails before publication -- for example because the actual -consuming checkout has moved past the evidence it was prepared from, or the -local graph was rebuilt in the meantime -- rolls the session back to its prior -prepared, unattached state on its own, keeping the failure reason visible in -status. The same session can then explicitly rebuild or rerun `prepare ---refresh` and attach again; it is never left pointing at a saved intent that -every retry would only re-fail. Once the trusted current pull request head has -genuinely moved past a saved intent, that old identity is retired before any -new evidence is authorized, so a checkout that has since moved on cannot block -its own recovery. This retirement, like the rollback above, is safe to retry -after an interruption partway through. +Before attempting a fresh reservation, the session first saves a durable +`reserving` marker -- proof that no GitHub write has happened yet, because +publication only ever follows a reservation that has itself already become +durable. If that reservation then fails within the same `attach` call -- for +example because the actual consuming checkout has moved past the evidence it +was prepared from, or the local graph was rebuilt in the meantime -- the same +call rolls the session back to its prior prepared, unattached state inline, +keeping the failure reason visible in status. If the process instead stops +before that same-call rollback, or before the reservation's own durable +transition out of `reserving` completes -- a crash, or a storage fault like +the one that also surfaces as a `ContextError` from a saved-state transition +that did not complete -- the saved intent is left as `reserving` instead. The +next `attach` recognizes a saved `reserving` marker, finishes abandoning it +and clearing the session back to its prior prepared, unattached state, and +asks the caller to rerun attach rather than completing a fresh attempt in +that same call, since resuming immediately could race a concurrent recovery. +Only once that cleanup has actually finished is the session safe to +explicitly rebuild, rerun `prepare --refresh`, or attach again; refresh must +not be used while a `reserving` cleanup is still outstanding, and a storage or +other fault surfaced while attach itself was trying to advance a saved intent +does not promise that this cleanup has already happened -- rerunning attach is +what finishes it. + +A saved `pending` or `uncertain` intent is different: its reservation is +already durable, and GitHub publication may have begun or even completed +before a lost response left the local state unconfirmed, so it is never +blanket-cleared the way a `reserving` marker is. Once the trusted current pull +request head has genuinely moved past such a saved intent, that old identity +is retired before any new evidence is authorized, so a checkout that has since +moved on cannot block its own recovery. This retirement, like the `reserving` +cleanup above, is safe to retry after an interruption partway through. The lower-level expert form remains available for scripts that intentionally manage request files and revisions: diff --git a/src/code_mower/context_guided.py b/src/code_mower/context_guided.py index 5d323985..b31bddc1 100644 --- a/src/code_mower/context_guided.py +++ b/src/code_mower/context_guided.py @@ -211,7 +211,13 @@ def _abandon_and_clear( Used only when ``record["revision"]`` is known to have not reached ``_finish_publication``'s remote publication check, so any stored delivery binding for it is never published and safe to abandon. Never - used to blanket-clear a pre-existing pending/uncertain retry. + used to blanket-clear a pre-existing pending/uncertain retry -- those + have already durably transitioned past ``reserving`` and may be the + GitHub-accepted state a lost response only looked like it missed. Both + the same-call rollback after a failed fresh reservation and a later + attach's recognition of a still-``reserving`` record call this; in + either case the identity being cleared is proven pre-publication by the + caller, never inferred here from a missing binding alone. ``record_failure`` may already have advanced the association generation, so the session is re-read here rather than trusting a stale one; if a concurrent operation has already moved the saved revision on, cleanup is @@ -272,6 +278,24 @@ def attach_session( record = current_record head, current = _remote_input(record["repo"], pr, token=token, authorities=authorities) + if record["attachment_state"] == "reserving": + # A ``reserving`` identity has, by construction, never reached + # ``_finish_publication``: it is written before ``reserve_attachment`` + # is even called, and only ever advances to ``pending`` -- still + # strictly before any GitHub write -- after that reservation + # durably succeeds. Finding it still ``reserving`` on a later + # attach therefore always means the same call that minted it + # stopped somewhere before, during, or just after reservation, + # with no public write possible, so it is always safe to abandon + # here regardless of the requested pull request. This never + # completes a fresh reattachment inline -- that would risk a + # public write racing this recovery -- so the caller reruns + # attach once cleanup below is done. + _abandon_and_clear(association_store, packet_store, record) + raise ContextError( + "a saved publication attempt did not complete; rerun attach to finish reconciliation" + ) + if record["attachment_state"] == "published": if record["pr"] != pr: raise ContextError("this session is already attached to a different pull request") @@ -353,7 +377,7 @@ def attach_session( expected_generation=record["generation"], changes={ "stage": "prepared", "pr": pr, "head": head, "revision": revision, - "attachment_state": "pending", + "attachment_state": "reserving", }, ) try: @@ -367,6 +391,21 @@ def attach_session( # way. _abandon_and_clear(association_store, packet_store, record) raise + # The reservation itself is now durable and unpublished. Recording + # that as ordinary ``pending`` -- still strictly before + # ``_finish_publication`` ever touches GitHub -- is what lets a later + # attach trust that ``pending`` always names a real, resumable + # reservation, never the ghost of one that was silently abandoned + # underneath it. If this transition itself does not complete (a + # crash, or a storage/generation failure), the record stays + # ``reserving`` and the next attach recognizes and clears it instead + # of replaying a reservation that may already be gone. + record = context_session.update( + association_store, + record["session_id"], + expected_generation=record["generation"], + changes={"attachment_state": "pending"}, + ) return _finish_publication( association_store, packet_store, diff --git a/src/code_mower/context_session.py b/src/code_mower/context_session.py index 54c097ed..32da18bb 100644 --- a/src/code_mower/context_session.py +++ b/src/code_mower/context_session.py @@ -19,7 +19,14 @@ ASSOCIATION_SCHEMA = "code_mower.contextSession.v1" STATUS_SCHEMA = "code_mower.contextSessionStatus.v1" STAGES = frozenset(("selected", "preparing", "prepared", "attached", "reviewed")) -ATTACHMENT_STATES = frozenset(("none", "pending", "published", "uncertain")) +ATTACHMENT_STATES = frozenset( + ("none", "reserving", "pending", "published", "uncertain") +) +#: ``reserving`` is the durable, pre-publication marker a freshly minted +#: attachment identity holds before ``reserve_attachment`` is even called. +#: It proves no GitHub write has begun, so it is always safe to abandon; +#: only a successful transition to ``pending`` (still before any GitHub +#: write) records that the reservation itself durably exists. CONTEXT_STATES = frozenset( ("unchecked", "ready", "expired", "authorization_failed", "unavailable") ) @@ -232,7 +239,7 @@ def validate(value: Mapping[str, Any]) -> dict[str, Any]: attachment_fields = (pr, head, revision) if attachment == "none" and any(item is not None for item in attachment_fields): raise ContextError("private session context revision has no attachment") - if attachment in {"pending", "uncertain"} and ( + if attachment in {"reserving", "pending", "uncertain"} and ( value["stage"] != "prepared" or any(item is None for item in attachment_fields) ): raise ContextError("private session context attachment intent is incomplete") @@ -476,7 +483,11 @@ def status(record: Mapping[str, Any] | None, *, lease_live: bool) -> dict[str, A else "Verify the selected connection, then rerun prepare with --refresh." ), } - if record["attachment_state"] == "pending": + if record["attachment_state"] in {"reserving", "pending"}: + # A ``reserving`` intent has never reached a GitHub write, but it is + # reported the same as ``pending`` here: both mean only attach can + # safely resolve the saved identity, and neither is a matter for + # owner judgement the way an uncertain publication result is. return { "schema": STATUS_SCHEMA, "selected": True, "configured": True, "stage": "attachment_pending", "dependent_work": "paused", diff --git a/tests/test_context_guided.py b/tests/test_context_guided.py index b31cfbb5..85d6054f 100644 --- a/tests/test_context_guided.py +++ b/tests/test_context_guided.py @@ -328,6 +328,13 @@ def test_process_crash_after_status_reuses_the_pending_revision(self): self.assertEqual(context_session.read(self.associations, self.session["id"])["revision"], revision) def test_process_crashes_around_local_reservation_and_publish_mark_resume(self): + # A crash inside ``reserve_attachment`` itself leaves the durable + # ``reserving`` marker behind with no binding ever created -- a + # process stop before reservation creates any binding + # (codex:1a4bc7b34687da726908). A later attach must recognize that + # state, safely finish cleanup, and only then let a fresh attach + # succeed; it must never silently resume toward publication in the + # same call that recognized the stale marker. with self.patches()[0], self.patches()[1], self.patches()[2], self.patches()[3], self.patches()[4], \ mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()): with self.assertRaises(KeyboardInterrupt): @@ -339,12 +346,21 @@ def test_process_crashes_around_local_reservation_and_publish_mark_resume(self): pr=42, backend=self.fixture.backend, ) - pending = context_session.read(self.associations, self.session["id"]) - revision = pending["revision"] - self.assertEqual(pending["attachment_state"], "pending") + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + report, code = self.attach() self.assertEqual((report["status"], code), ("attached", 0)) - self.assertEqual(context_session.read(self.associations, self.session["id"])["revision"], revision) # Move to a new head so the same session creates another intent. GitHub # accepts its comment and the binding is enabled, then the process stops @@ -558,10 +574,11 @@ def racing_record_failure(store, record, error): ("none", None, None, None), ) - def test_rollback_failure_retains_the_pending_intent_rather_than_claiming_success(self): - """If the abandon step itself fails, the saved pending intent is left - in place rather than reported as cleared -- a ``ContextError`` there - does not prove no local write occurred (codex:1a4bc7b34687da726908).""" + def test_rollback_failure_retains_the_reserving_intent_rather_than_claiming_success(self): + """If the abandon step itself fails, the saved durable pre-publication + (``reserving``) intent is left in place rather than reported as + cleared -- a ``ContextError`` there does not prove no local write + occurred (codex:1a4bc7b34687da726908).""" with ( self.patches()[0], self.patches()[1], self.patches()[2], self.patches()[3], self.patches()[4], @@ -573,9 +590,154 @@ def test_rollback_failure_retains_the_pending_intent_rather_than_claiming_succes self.associations, self.fixture.store, self.record, repo_path=self.root, pr=42, backend=self.fixture.backend, ) - pending = context_session.read(self.associations, self.session["id"]) - self.assertEqual(pending["attachment_state"], "pending") - self.assertIsNotNone(pending["revision"]) + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + def test_interrupted_association_clear_after_reservation_recovers_on_retry(self): + """A crash between abandoning an unpublished binding and clearing the + session association leaves the durable ``reserving`` marker behind, + with the reservation it once named already gone. The next attach + recognizes that state, finishes cleanup idempotently, and only then + permits an explicit refresh and a successful reattachment + (codex:1a4bc7b34687da726908).""" + revision = uuid.uuid4().hex + reserving = context_session.update( + self.associations, self.session["id"], expected_generation=self.record["generation"], + changes={ + "stage": "prepared", "pr": 42, "head": self.head, "revision": revision, + "attachment_state": "reserving", + }, + ) + context_guided.reserve_attachment( + self.fixture.store, reserving["connection"], reserving["packet"], reserving["policy"], + context_guided.ContextRequest(reserving["repo"], reserving["work_item"], "codex:orchestrator"), + pr=42, head=self.head, revision=revision, backend=self.fixture.backend, + ) + # The reservation genuinely exists on disk; a crash then lands + # between the two writes ``_abandon_and_clear`` itself makes. + context_guided.abandon_attachment( + self.fixture.store, reserving["connection"], reserving["packet"], revision, + ) + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + + def test_concurrent_failure_after_cleanup_read_preserves_the_diagnostic(self): + """A ``record_failure`` that lands after ``_abandon_and_clear`` reads + the association but before it clears it -- bumping the generation + while keeping the same revision -- must not be lost: the same + revision still clears safely on a later attach while the latest + failure diagnostic survives (codex:1a4bc7b34687da726908).""" + calls = {"n": 0} + real_read = context_session.read + + def racing_read(store, session_id): + calls["n"] += 1 + current = real_read(store, session_id) + if calls["n"] == 2: + context_session.update( + store, session_id, expected_generation=current["generation"], + changes={"context_state": "authorization_failed"}, + ) + return current + + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=ContextError("boom")), + mock.patch.object(context_session, "read", side_effect=racing_read), + ): + with self.assertRaisesRegex(ContextError, "boom"): + context_guided.attach_session( + self.associations, self.fixture.store, self.record, repo_path=self.root, + pr=42, backend=self.fixture.backend, + ) + stranded = context_session.read(self.associations, self.session["id"]) + self.assertEqual(stranded["attachment_state"], "reserving") + self.assertEqual(stranded["context_state"], "authorization_failed") + revision = stranded["revision"] + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + # The failure diagnostic recorded during the race is preserved + # rather than silently erased by the eventual cleanup. + self.assertEqual(recovered["context_state"], "authorization_failed") + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + + def test_abandon_and_clear_never_touches_a_different_or_published_revision(self): + """The identity check the reserving cleanup relies on must never + clear a different, replaced, or already-published revision, even + under a stale generation (codex:1a4bc7b34687da726908).""" + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + published = context_session.read(self.associations, self.session["id"]) + stale = {**published, "revision": "0" * 32} + context_guided._abandon_and_clear(self.associations, self.fixture.store, stale) + unchanged = context_session.read(self.associations, self.session["id"]) + self.assertEqual(unchanged, published) + + def test_interrupted_pending_transition_recovers_without_public_writes(self): + """A crash after a fresh reservation succeeds but before its session + association durably transitions to ``pending`` -- still strictly + before any GitHub write -- must recover on retry by abandoning the + unpublished binding and restoring prepared/none, with zero public + writes, rather than resuming toward publication in that same + recovery (codex:1a4bc7b34687da726908).""" + real_update = context_session.update + + def failing_transition(store, session_id, *, expected_generation, changes): + if changes == {"attachment_state": "pending"}: + raise OSError("disk full") + return real_update(store, session_id, expected_generation=expected_generation, changes=changes) + + with mock.patch.object(context_session, "update", side_effect=failing_transition): + # The mocked ``OSError`` never reaches the store boundary that + # would normally raise it, but it still propagates out through + # the surrounding ``association_store.locked`` block that + # ``attach_session`` holds for its whole body, so it is normalized + # to the same fail-closed public ``ContextError`` a genuine + # storage fault there would produce. + with self.assertRaisesRegex(ContextError, "private context store is unavailable or unsafe"): + self.attach() + reserving = context_session.read(self.associations, self.session["id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + revision = reserving["revision"] + # The reservation itself genuinely completed before the interrupted + # transition -- a real, unpublished binding exists. + self.assertFalse(read_binding(self.fixture.store, revision)["published"]) + self.assertEqual(self.comments, []) + self.assertEqual(self.events, []) + + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.session["id"]) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + self.assertEqual(self.events, []) + with self.assertRaises(ContextError): + read_binding(self.fixture.store, revision) + + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) def test_provider_free_cross_process_qualification_is_symmetric_for_both_hosts(self): delivered = [] @@ -926,11 +1088,16 @@ def test_same_head_failures_recover_without_stranding_refresh(self): def test_pending_retry_revalidates_the_actual_consumer_before_publication(self): """A resumed pending/uncertain attachment must revalidate the current consumer rather than replay a stale reservation, while preserving the - saved intent and its failure diagnostics (codex:65a17212478a56416b1c).""" + saved intent and its failure diagnostics (codex:65a17212478a56416b1c). + The crash is injected at the start of ``_finish_publication`` so the + reservation has already durably transitioned to ``pending`` -- still + strictly before any GitHub write -- rather than leaving the + pre-publication ``reserving`` marker this same scenario would leave + if the crash instead landed inside ``reserve_attachment`` itself.""" with ( self.patches()[0], self.patches()[1], self.patches()[2], self.patches()[3], self.patches()[4], - mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()), + mock.patch.object(context_guided, "_finish_publication", side_effect=KeyboardInterrupt()), ): with self.assertRaises(KeyboardInterrupt): context_guided.attach_session( @@ -949,6 +1116,37 @@ def test_pending_retry_revalidates_the_actual_consumer_before_publication(self): self.assertNotEqual(after["context_state"], "ready") self.assertEqual(self.comments, []) + def test_reserving_retry_recovers_before_any_reservation_ever_completes(self): + """A crash inside ``reserve_attachment`` itself -- before the saved + intent ever durably transitions to ``pending`` -- leaves the durable + pre-publication ``reserving`` marker instead. A later attach must + recognize it, safely abandon the identity (idempotently, since no + binding was ever created), and restore prepared/none rather than + replaying it as if it were a genuine pending publication attempt + (codex:1a4bc7b34687da726908).""" + with ( + self.patches()[0], self.patches()[1], self.patches()[2], + self.patches()[3], self.patches()[4], + mock.patch.object(context_guided, "reserve_attachment", side_effect=KeyboardInterrupt()), + ): + with self.assertRaises(KeyboardInterrupt): + context_guided.attach_session( + self.associations, self.store, self.record, repo_path=self.repository, + pr=1, backend=None, + ) + reserving = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual(reserving["attachment_state"], "reserving") + with self.assertRaisesRegex(ContextError, "rerun attach"): + self.attach() + recovered = context_session.read(self.associations, self.SESSION_ID) + self.assertEqual( + (recovered["attachment_state"], recovered["pr"], recovered["head"], recovered["revision"]), + ("none", None, None, None), + ) + self.assertEqual(self.comments, []) + report, code = self.attach() + self.assertEqual((report["status"], code), ("attached", 0)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_context_session.py b/tests/test_context_session.py index ad0cce1c..dbe2f8c2 100644 --- a/tests/test_context_session.py +++ b/tests/test_context_session.py @@ -178,6 +178,45 @@ def test_resolution_rejects_conflicting_trusted_values(self): with self.assertRaisesRegex(ContextError, "conflicts with the saved session"): context_session.resolve_bound("repository", "owner/repo", "owner/other") + def test_reserving_state_is_reported_like_pending_and_rejects_malformed_combinations(self): + """The durable pre-publication ``reserving`` marker validates and is + reported exactly like ``pending`` -- attach reconciliation, no owner + action -- and malformed cross-state combinations fail closed rather + than being inferred from a missing artifact (codex:1a4bc7b34687da726908).""" + record = context_session.create(self.store, self.session, work_item="SECRET-1", policy=POLICY) + prepared = context_session.update( + self.store, self.session["id"], expected_generation=record["generation"], + changes={ + "stage": "prepared", "builder": "codex", "query_mode": "work_item", + "request_hash": "c" * 64, "packet": "b" * 32, "work_order": "work-order.md", + }, + ) + reserving = context_session.update( + self.store, self.session["id"], expected_generation=prepared["generation"], + changes={ + "pr": 7, "head": "d" * 40, "revision": "e" * 32, "attachment_state": "reserving", + }, + ) + self.assertEqual(reserving["attachment_state"], "reserving") + status = context_session.status(reserving, lease_live=True) + self.assertEqual(status["stage"], "attachment_pending") + self.assertFalse(status["owner_action"]) + self.assertIn("attach", status["next_action"]) + for private in ("SECRET-1", "example-context", "owner/repo", "e" * 32): + self.assertNotIn(private, json.dumps(status)) + + for mutation in ( + # Claims a mid-preparation stage while still holding an + # attachment identity -- a malformed cross-state record. + {"stage": "preparing", "builder": None, "query_mode": None, "request_hash": None, + "packet": None, "work_order": None}, + {"pr": None}, + {"head": None}, + {"revision": None}, + ): + with self.subTest(mutation=mutation), self.assertRaises(ContextError): + context_session.validate({**copy.deepcopy(reserving), **mutation}) + def test_malformed_or_cross_bound_records_fail_closed(self): record = context_session.create(self.store, self.session, work_item="ITEM-1", policy=POLICY) mutations = ( From 9e64f17de2bd4fe8c07aabdd5e0e8debe9ee8976 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 01:58:49 -0700 Subject: [PATCH 32/33] Complete reserving state integration (codex:1a4bc7b34687da726908) context_prepare.prepare's earliest attachment-in-progress guard only checked pending/uncertain, so a saved reserving association could reach ordinary prepare or --refresh and have its unpublished binding replaced or discarded instead of preserved for attach's cleanup. context_session .status also checked context_state failure values before the attachment branch, so a failed reservation could report optional/refresh guidance instead of attach reconciliation even though reserving proves no GitHub write ever began. Add reserving to prepare's guard alongside pending/uncertain, and move the reserving check in status ahead of the context_state failure block (pending/uncertain keep their existing failure precedence). --- src/code_mower/context_prepare.py | 2 +- src/code_mower/context_session.py | 18 ++++++-- tests/test_context_prepare.py | 77 ++++++++++++++++++++++++++++++- tests/test_context_session.py | 71 ++++++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/src/code_mower/context_prepare.py b/src/code_mower/context_prepare.py index a4295c47..a15b3464 100644 --- a/src/code_mower/context_prepare.py +++ b/src/code_mower/context_prepare.py @@ -225,7 +225,7 @@ def prepare( work_order=record["work_order"], reused=True, ), 0 - if record["attachment_state"] in {"pending", "uncertain"}: + if record["attachment_state"] in {"reserving", "pending", "uncertain"}: if refresh: raise ContextError( "reconcile the saved attachment before refreshing or start a new session" diff --git a/src/code_mower/context_session.py b/src/code_mower/context_session.py index 32da18bb..4a4599ad 100644 --- a/src/code_mower/context_session.py +++ b/src/code_mower/context_session.py @@ -442,6 +442,18 @@ def status(record: Mapping[str, Any] | None, *, lease_live: bool) -> dict[str, A "stage": "not_configured", "dependent_work": "usable", "owner_action": False, "next_action": "Continue the ordinary workflow or configure an optional context connection.", } + if record["attachment_state"] == "reserving": + # A ``reserving`` intent has never reached a GitHub write, so attach + # can always safely reconcile it without reauthorization; that takes + # precedence over any ``context_state`` failure recorded against a + # prior generation of this same record, unlike ``pending``/ + # ``uncertain`` below, whose failures may still need owner action. + return { + "schema": STATUS_SCHEMA, "selected": True, "configured": True, + "stage": "attachment_pending", "dependent_work": "paused", + "owner_action": False, + "next_action": "Rerun attach to reconcile or finish the saved publication intent.", + } if record["context_state"] in {"expired", "authorization_failed", "unavailable"}: expired = record["context_state"] == "expired" authorization = record["context_state"] == "authorization_failed" @@ -483,11 +495,7 @@ def status(record: Mapping[str, Any] | None, *, lease_live: bool) -> dict[str, A else "Verify the selected connection, then rerun prepare with --refresh." ), } - if record["attachment_state"] in {"reserving", "pending"}: - # A ``reserving`` intent has never reached a GitHub write, but it is - # reported the same as ``pending`` here: both mean only attach can - # safely resolve the saved identity, and neither is a matter for - # owner judgement the way an uncertain publication result is. + if record["attachment_state"] == "pending": return { "schema": STATUS_SCHEMA, "selected": True, "configured": True, "stage": "attachment_pending", "dependent_work": "paused", diff --git a/tests/test_context_prepare.py b/tests/test_context_prepare.py index 1c34fee4..5a6f01ba 100644 --- a/tests/test_context_prepare.py +++ b/tests/test_context_prepare.py @@ -11,7 +11,7 @@ from pathlib import Path from unittest import mock -from code_mower import context_prepare, context_session, session +from code_mower import context_guided, context_prepare, context_session, session from code_mower.context_connections import connect from code_mower.context_contract import ContextError, ContextRetrievalError from code_mower.context_store import ContextStore @@ -149,6 +149,81 @@ def test_prepare_preserves_a_pending_or_uncertain_attachment_for_reconciliation( self.backend.revoked = False self.assertEqual(prepared["packet"], saved["packet"]) + def test_prepare_preserves_a_genuine_unpublished_reservation_and_refuses_refresh(self): + """A real ``reserving`` intent -- minted by attach and left behind by + a crash inside ``reserve_attachment`` itself, strictly before any + GitHub write -- must guard prepare exactly like ``pending``/ + ``uncertain``: the earliest check, before checkout revision, query + fingerprint, work-order, packet store, or provider access. Neither + ordinary prepare nor a changed-query ``--refresh`` may inspect or + abandon the saved binding; only attach-style cleanup may resolve it, + and only then does prepare work again (codex:1a4bc7b34687da726908).""" + record = self.create_record() + _report, code = self.prepare(record) + self.assertEqual(code, 0) + prepared = context_session.read(self.associations, record["session_id"]) + + with mock.patch.object( + context_guided, "_github_access", return_value=("token", ("controller",)) + ), mock.patch.object( + context_guided, "fetch_pull_request", return_value={"head": {"sha": "a" * 40}} + ), mock.patch.object( + context_guided, "reserve_attachment", side_effect=KeyboardInterrupt() + ): + with self.assertRaises(KeyboardInterrupt): + context_guided.attach_session( + self.associations, self.packet_store, prepared, + repo_path=self.repo, pr=42, backend=self.backend, + ) + reserving = context_session.read(self.associations, record["session_id"]) + self.assertEqual(reserving["attachment_state"], "reserving") + self.assertIsNotNone(reserving["revision"]) + + def snapshot(): + return { + path: path.read_bytes() + for path in sorted(self.private.rglob("*")) + if path.is_file() + } + + before = snapshot() + self.backend.revoked = True + + report, code = self.prepare(reserving) + self.assertEqual( + (code, report["status"], report["stage"], report["dependent_work"]), + (0, "attachment_in_progress", "attachment_pending", "paused"), + ) + self.assertEqual(self.backend.searches, 1) + self.assertEqual(snapshot(), before) + + with self.assertRaisesRegex(ContextError, "reconcile the saved attachment"): + self.prepare(reserving, refresh=True, query="a materially different bounded query") + self.assertEqual(self.backend.searches, 1) + self.assertEqual(snapshot(), before) + + self.backend.revoked = False + saved = context_session.read(self.associations, record["session_id"]) + self.assertEqual(saved, reserving) + self.assertEqual(prepared["packet"], saved["packet"]) + + with mock.patch.object( + context_guided, "_github_access", return_value=("token", ("controller",)) + ), mock.patch.object( + context_guided, "fetch_pull_request", return_value={"head": {"sha": "a" * 40}} + ): + with self.assertRaisesRegex(ContextError, "rerun attach to finish reconciliation"): + context_guided.attach_session( + self.associations, self.packet_store, saved, + repo_path=self.repo, pr=42, backend=self.backend, + ) + cleared = context_session.read(self.associations, record["session_id"]) + self.assertEqual(cleared["attachment_state"], "none") + + resumed, code = self.prepare(cleared) + self.assertEqual((code, resumed["status"], resumed["reused"]), (0, "prepared", True)) + self.assertEqual(self.backend.searches, 1) + def test_failed_or_interrupted_search_requires_explicit_refresh(self): record = self.create_record() self.backend.fail_search = True diff --git a/tests/test_context_session.py b/tests/test_context_session.py index dbe2f8c2..32569f1e 100644 --- a/tests/test_context_session.py +++ b/tests/test_context_session.py @@ -217,6 +217,77 @@ def test_reserving_state_is_reported_like_pending_and_rejects_malformed_combinat with self.subTest(mutation=mutation), self.assertRaises(ContextError): context_session.validate({**copy.deepcopy(reserving), **mutation}) + def test_reserving_precedes_context_state_failure_while_pending_and_uncertain_do_not(self): + """A ``reserving`` intent proves publication never began, so attach + reconciliation is safe without reauthorization and is reported that + way regardless of any ``context_state`` failure recorded against a + prior generation, or of required/optional policy. ``pending`` and + ``uncertain`` keep their existing failure precedence unchanged -- + their authorization failures may still need owner action + (codex:1a4bc7b34687da726908).""" + failure_stage_for = { + "ready": None, + "unchecked": None, + "unavailable": "context_unavailable", + "authorization_failed": "authorization_failed", + "expired": "context_expired", + } + counter = 0 + + def make_record(attachment_state, context_state, required): + nonlocal counter + counter += 1 + session_id = format(counter, "032x") + record = context_session.create( + self.store, session_value(session_id), work_item="SECRET-1", + policy={**POLICY, "required": required}, + ) + prepared = context_session.update( + self.store, session_id, expected_generation=record["generation"], + changes={ + "stage": "prepared", "builder": "codex", "query_mode": "work_item", + "request_hash": "c" * 64, "packet": "b" * 32, + "work_order": "work-order.md", "context_state": context_state, + }, + ) + return context_session.update( + self.store, session_id, expected_generation=prepared["generation"], + changes={ + "pr": 7, "head": "d" * 40, "revision": "e" * 32, + "attachment_state": attachment_state, + }, + ) + + for required in (True, False): + for context_state, failure_stage in failure_stage_for.items(): + with self.subTest(required=required, context_state=context_state, attachment="reserving"): + reserving = make_record("reserving", context_state, required) + status = context_session.status(reserving, lease_live=True) + self.assertEqual(status["stage"], "attachment_pending") + self.assertEqual(status["dependent_work"], "paused") + self.assertFalse(status["owner_action"]) + self.assertIn("attach", status["next_action"]) + for private in ("SECRET-1", "example-context", "owner/repo", "e" * 32): + self.assertNotIn(private, json.dumps(status)) + + for attachment in ("pending", "uncertain"): + with self.subTest(required=required, context_state=context_state, attachment=attachment): + record = make_record(attachment, context_state, required) + status = context_session.status(record, lease_live=True) + if failure_stage is not None: + self.assertEqual(status["stage"], failure_stage) + self.assertTrue(status["owner_action"]) + self.assertEqual( + status["dependent_work"], "paused" if required else "usable", + ) + else: + self.assertEqual( + status["stage"], + "attachment_uncertain" if attachment == "uncertain" else "attachment_pending", + ) + self.assertEqual(status["dependent_work"], "paused") + self.assertEqual(status["owner_action"], attachment == "uncertain") + def test_malformed_or_cross_bound_records_fail_closed(self): record = context_session.create(self.store, self.session, work_item="ITEM-1", policy=POLICY) mutations = ( From 9216db200e636089dc6baba555d7d1d28b9cf3d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 02:03:29 -0700 Subject: [PATCH 33/33] Use local reservation in prepare recovery test (codex:1a4bc7b34687da726908) --- tests/test_context_prepare.py | 77 ++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/tests/test_context_prepare.py b/tests/test_context_prepare.py index 5a6f01ba..ec3c0359 100644 --- a/tests/test_context_prepare.py +++ b/tests/test_context_prepare.py @@ -5,15 +5,17 @@ import io import json import os +import secrets import tempfile import unittest +import uuid from contextlib import contextmanager, redirect_stderr, redirect_stdout from pathlib import Path from unittest import mock -from code_mower import context_guided, context_prepare, context_session, session +from code_mower import context_delivery, context_guided, context_prepare, context_session, session from code_mower.context_connections import connect -from code_mower.context_contract import ContextError, ContextRetrievalError +from code_mower.context_contract import ContextError, ContextRequest, ContextRetrievalError from code_mower.context_store import ContextStore from test_context_connections import MemoryVault from test_context_packets import RetrievalBackend @@ -150,32 +152,48 @@ def test_prepare_preserves_a_pending_or_uncertain_attachment_for_reconciliation( self.assertEqual(prepared["packet"], saved["packet"]) def test_prepare_preserves_a_genuine_unpublished_reservation_and_refuses_refresh(self): - """A real ``reserving`` intent -- minted by attach and left behind by - a crash inside ``reserve_attachment`` itself, strictly before any - GitHub write -- must guard prepare exactly like ``pending``/ - ``uncertain``: the earliest check, before checkout revision, query - fingerprint, work-order, packet store, or provider access. Neither - ordinary prepare nor a changed-query ``--refresh`` may inspect or - abandon the saved binding; only attach-style cleanup may resolve it, - and only then does prepare work again (codex:1a4bc7b34687da726908).""" + """A real ``reserving`` intent -- exactly the durable, unpublished + state a crash inside ``reserve_attachment`` itself would leave + behind, strictly before any GitHub write -- must guard prepare + exactly like ``pending``/``uncertain``: the earliest check, before + checkout revision, query fingerprint, work-order, packet store, or + provider access. Neither ordinary prepare nor a changed-query + ``--refresh`` may inspect or abandon the saved binding; only + attach's own cleanup primitive may resolve it, and only then does + prepare work again (codex:1a4bc7b34687da726908).""" record = self.create_record() _report, code = self.prepare(record) self.assertEqual(code, 0) prepared = context_session.read(self.associations, record["session_id"]) - with mock.patch.object( - context_guided, "_github_access", return_value=("token", ("controller",)) - ), mock.patch.object( - context_guided, "fetch_pull_request", return_value={"head": {"sha": "a" * 40}} - ), mock.patch.object( - context_guided, "reserve_attachment", side_effect=KeyboardInterrupt() - ): - with self.assertRaises(KeyboardInterrupt): - context_guided.attach_session( - self.associations, self.packet_store, prepared, - repo_path=self.repo, pr=42, backend=self.backend, - ) - reserving = context_session.read(self.associations, record["session_id"]) + revision = uuid.uuid4().hex + head = secrets.token_hex(20) + reserving = context_session.update( + self.associations, + prepared["session_id"], + expected_generation=prepared["generation"], + changes={ + "pr": 42, "head": head, "revision": revision, + "attachment_state": "reserving", + }, + ) + # This organization connection's evidence versions independently of + # any Git checkout, so the reservation names no consuming revision -- + # the same "no Git checkout revision" call attach would have made. + context_delivery.reserve_attachment( + self.packet_store, + reserving["connection"], + reserving["packet"], + reserving["policy"], + ContextRequest(reserving["repo"], reserving["work_item"], reserving["host"] + ":orchestrator"), + pr=42, + head=head, + revision=revision, + consuming_revision=None, + backend=self.backend, + ) + binding = context_delivery.read_binding(self.packet_store, revision) + self.assertFalse(binding["published"]) self.assertEqual(reserving["attachment_state"], "reserving") self.assertIsNotNone(reserving["revision"]) @@ -207,18 +225,11 @@ def snapshot(): self.assertEqual(saved, reserving) self.assertEqual(prepared["packet"], saved["packet"]) - with mock.patch.object( - context_guided, "_github_access", return_value=("token", ("controller",)) - ), mock.patch.object( - context_guided, "fetch_pull_request", return_value={"head": {"sha": "a" * 40}} - ): - with self.assertRaisesRegex(ContextError, "rerun attach to finish reconciliation"): - context_guided.attach_session( - self.associations, self.packet_store, saved, - repo_path=self.repo, pr=42, backend=self.backend, - ) + context_guided._abandon_and_clear(self.associations, self.packet_store, saved) cleared = context_session.read(self.associations, record["session_id"]) self.assertEqual(cleared["attachment_state"], "none") + with self.assertRaises(ContextError): + context_delivery.read_binding(self.packet_store, revision) resumed, code = self.prepare(cleared) self.assertEqual((code, resumed["status"], resumed["reused"]), (0, "prepared", True))