diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65868d1d..d8772e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,87 @@ jobs: --timeout 240 --json + graph_containment: + # The local-graph boundary, against the real kernel mechanism rather than a + # stand-in for one. The unit suite skips these when a host offers no + # mechanism, which is right for a laptop and useless as coverage: this job + # installs bubblewrap and sets CODE_MOWER_REQUIRE_CONTAINMENT, which turns + # that skip into a failure. + name: graph containment + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.12" + + - name: Install bubblewrap + run: | + sudo apt-get update + sudo apt-get install -y bubblewrap + test -x /usr/bin/bwrap + + # Ubuntu 24.04 restricts unprivileged user namespaces by default, which + # is what bubblewrap needs to build a mount namespace without being + # setuid. Enabling it is a property of this runner, not of the product: + # a host that refuses keeps refusing builds, which is the fail-closed + # posture. Reported rather than asserted, so the suite below is what + # decides the job. + - name: Allow unprivileged user namespaces + run: | + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true + sudo sysctl -w kernel.unprivileged_userns_clone=1 || true + bwrap --unshare-net --dev-bind / / /bin/true \ + && echo "bwrap: namespaces available" \ + || echo "bwrap: refused a namespace on this runner" + + - name: Install package + run: python -m pip install -e . + + - name: Real containment tests + env: + CODE_MOWER_REQUIRE_CONTAINMENT: "1" + run: python -m unittest discover -s tests -p test_context_graph_lifecycle.py -v + + graph_containment_macos: + # The Seatbelt half of the same claim. The bubblewrap job above proves the + # Linux boundary and nothing whatever about this one, and macOS is the + # platform this tool is developed on: leaving the profile to be exercised + # only by whoever happens to run the suite on a laptop is how it stayed + # unexecuted. ``sandbox-exec`` ships with the OS, so there is nothing to + # install -- the job is the evidence that the profile runs at all. + # + # Required, like the bubblewrap job. macOS containment is a behaviour claim + # in ``docs/context-graph-lifecycle.md``, and a claim whose only check is + # allowed to be red is not being checked. A red result still carries the + # probe's own diagnosis of which candidate failed and what the launcher + # said, which is what turned the equivalent bubblewrap failure into a + # one-round fix. + name: graph containment (macOS) + runs-on: macos-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.12" + + - name: Confirm the system sandbox is present + run: test -x /usr/bin/sandbox-exec + + - name: Install package + run: python -m pip install -e . + + - name: Real containment tests + env: + CODE_MOWER_REQUIRE_CONTAINMENT: "1" + run: python -m unittest discover -s tests -p test_context_graph_lifecycle.py -v + package: name: package runs-on: ubuntu-latest diff --git a/docs/context-graph-lifecycle.md b/docs/context-graph-lifecycle.md new file mode 100644 index 00000000..21240dae --- /dev/null +++ b/docs/context-graph-lifecycle.md @@ -0,0 +1,568 @@ +# Local repository graph: revision-bound lifecycle + +Implements [issue #913](https://github.com/codemower-ai/code-mower/issues/913) +under [epic #902](https://github.com/codemower-ai/code-mower/issues/902), on top +of the adopt decision recorded in [Graphify evaluation](graphify-evaluation.md). + +This is the safe lifecycle around an optional local code graph: how one gets +built, what it is allowed to see, where it is kept, and when a consumer must +refuse it. It adds no dependency, no background service, no hook, no watcher, +and no default indexing step. Nothing on a default install path builds, reads, +or requires a graph. + +Two modules divide the work: + +- `context_graph.py` (from #876) decides whether a delivered packet's + **citations** are in scope and fresh enough to use. +- `context_graph_lifecycle.py` (this document) decides whether the graph + **should have existed** — which revision it binds, which bytes produced it, + where its state lives, and when it fails closed. + +## The problem + +A graph indexer pointed at a working checkout is unsafe in two directions. It +reads files nobody agreed to index — untracked scratch files, ignored +`.env` files, another worktree reached through a symlink — and it produces an +artifact with no way to tell which revision it describes, so a graph built three +commits ago answers today's question with yesterday's code and looks identical +to a fresh one. + +Constraint 1 in the evaluation is the sharp edge: the provider owns no +provenance at all. If Code Mower does not bind the revision, nothing does. + +## What a build does + +`build_graph()` is the only way a generation is created. In order: + +1. **Resolve the revision.** The full commit and tree object names, never an + abbreviation and never a branch name. The tree is resolved separately + because it is what a consumer actually compares. +2. **Take the tracked census.** `git ls-tree -r` against the *commit*, not the + working tree and not the index. Symlinks (`120000`) and submodules + (`160000`) are skipped and recorded as skipped, because a symlink can name a + target the build was never shown and a gitlink names a commit in a + repository it was never authorized to read. Committed private state is + skipped for a third reason, at any depth and case-folded: the roots are + `context_graph`'s excluded roots themselves — `.git`, `.graph`, `.graphify`, + `.code-mower` — bound rather than copied, so the set that refuses a citation + into private state is the same set that keeps those bytes away from the + indexer. A tracked `.graphify/` or `.graph/` is somebody's old index, and + materializing it would let the provider resume from a cache built over + content this build never saw, and let the adapter collect tracked repository + bytes as if the provider had just produced them. A tracked `.code-mower/` + is this tool's own packets and evidence, which the evidence contract refuses + to let a packet cite and which therefore may not be indexed either. The + census digest covers mode, blob name, size and path + for every entry in sorted order. Both halves of the census are bounded as + they are collected: the file-count budget covers what is materialized, and a + matching budget covers what is skipped, because a repository of symlinks, + submodules, or committed private state grows the skipped list without adding + a single entry to the other one. +3. **Materialize into private state.** Each blob is written into a fresh 0700 + directory as a 0600 file. Untracked and ignored files have no path into the + graph because they are never written, rather than because something filtered + them out afterwards. +4. **Run the indexer with a scrubbed environment, inside a network-denying + sandbox.** The provider process inherits an allowlist — `PATH`, `TMPDIR`, + `LANG`, `LC_ALL`, `TZ` — and nothing else, with `HOME` and the XDG + directories pointing into the build's own scratch area. A newly invented + secret variable is excluded by default because the list names what is kept, + not what is dropped. The network boundary is separate and is described + below; the emptied proxy variables are hygiene, not that boundary. +5. **Publish atomically.** The manifest is serialized and read back through the + same validation every consumer applies before anything is written: a + manifest this process can write but no reader can load would otherwise + become `current`, prune the generation that worked, and read back `invalid` + on the next status — a build reporting success while destroying the only + usable graph. Then the generation is assembled under a staging name, + fsynced, renamed into `generations/`, and only then does the `current` + pointer start naming it. A reader sees the whole previous generation or the + whole new one. + +Publication and pruning happen inside one locked critical section. Two builders +that race are serialized, and neither can delete the generation the other just +published while `current` still names it. + +`remove` takes the same lock. A removal running beside a build would otherwise +delete its materialized sources, its output, and the generations directory, and +the builder would then either fail or recreate state that `remove` had already +reported as gone. The lock file lives *beside* the state directory rather than +inside it, so it survives the removal it serializes: a lock inside the deleted +tree would be unlinked mid-removal, and the next builder would create a new +inode and hold a lock nobody else was waiting on. What is left behind is an +empty 0600 file carrying nothing. + +Readers take no lock at all, so a refresh can publish and prune between the +moment `graph_status()` reads the `current` pointer and the moment it finishes +validating what that pointer named. A failing verdict is therefore confirmed +against the pointer before it is returned, and a pointer that moved is read +again — otherwise a healthy refresh would surface as `invalid` or `corrupt`. A +verdict about the generation `current` still names is returned as it stands; the +retry is for a moved pointer, not a poll. + +## The containment boundary + +An environment variable is a request, not a boundary: `NO_PROXY=*` asks a +cooperating client to connect *directly*, and on a host with internet access an +uncooperative provider is unaffected by any of it. A working directory is not a +boundary either: pointing a provider at the materialized copy does not stop it +reading the checkout next door. So the provider is launched behind an argv +prefix that denies it, at the operating-system level, both the network and +every path outside the build's own directories — `sandbox-exec` with a +`(deny default)` profile on macOS, `bwrap` with an empty new root on Linux. + +What the child can see is the whole of it: the materialized copy and the +build's redirected `HOME` and `TMPDIR`, writable; the pinned provider's own +install and the system runtime it needs to start, read-only. The operator's +home, their other checkouts, and every ignored `.env` beside them are not +unreadable — they are absent. `unshare --net` used to be a candidate and is +gone: it denies the network and leaves the host filesystem in place, which is +half a boundary. + +"The system runtime" is a list of runtime directories, not a list of top-level +ones. It used to say `/usr`, `/etc` and `/Library`, which claims far more than +"what a runtime needs to start": `/usr` carries `/usr/local` — Homebrew's whole +prefix, its `etc` and `var` included — and `/usr/src`, either of which can hold +a checkout; `/etc` carries whatever service credentials a host's packages left +world-readable; and `/Library` carries `Keychains`, `Preferences`, +`Application Support` and the rest of a Mac's machine-wide operator data. Now it +names the loader and C library directories, the system binary directories, +`/usr/share`, Apple's signed `/System` volume, the dyld and time-zone databases, +the two `/Library` paths that hold a runtime rather than operator data +(`/Library/Frameworks` and the command line tools' own bundle), and `/etc` one +entry at a time — the loader's cache and configuration, the time zone, the +account databases, OpenSSL's configuration file. Nothing else. A host whose +runtime needs something outside that list refuses builds, because the probe +cannot start a child under the boundary; it does not get a wider boundary. + +The refusals are applied to the **whole readable set**, not only to each root a +build asks for. Every exposure a build derives is checked as it is derived, but +the runtime above is added afterwards and unconditionally, so the set the child +is really confined to was never examined as a set — which is how a runtime path +that contained the checkout could leave the live working tree readable beside +the materialized copy that exists to replace it, with every per-root check +passing. A base interpreter prefix wide enough to contain the runtime is handled +the same way from the other side: `/usr`, which is what an environment created +from the system Python records, is narrowed to the runtime directories inside it +rather than exposed whole, and on a prefix like that one those are already what +every child gets, so the exposure does not grow at all. + +"The provider's own install" is a layout this module *proves* rather than one +it infers from depth. It used to expose the executable's parent and +grandparent, on the reasoning that a console script lives in a virtual +environment's `bin`; where that reasoning is wrong it is wrong in the widest +possible direction, because `~/bin/graphify` makes the grandparent the +operator's entire home, `/opt/graphify` makes it `/`, and a provider installed +inside the checkout makes it the live working tree the materialized copy exists +to keep away from the provider. So an install root is now a directory holding a +`pyvenv.cfg` whose script directory holds the executable — a virtual +environment, which is what pinning a provider produces — and the root arrived +at is refused outright if it is the filesystem root, the operator's home, the +checkout being indexed, or an ancestor of either. A provider already inside the +read-only system runtime asks for no extra exposure and gets none. Anything +else is refused with an instruction to pin the provider into its own +environment, rather than exposed as a guess. + +A virtual environment is not self-contained: it reaches its base interpreter +and standard library through a link out of its own `bin`, so that base +installation is exposed alongside it. *Which* base is read out of the +environment's own `pyvenv.cfg` — the `base-prefix`, `base-executable`, +`executable` and `home` records that `venv`, `virtualenv` and `uv` write — and +not taken from `sys.base_prefix`, which names the interpreter running Code +Mower. The two are the same installation only when the provider happened to be +pinned with this process's Python; pin it with a `uv`-managed or otherwise +separately installed one, as is entirely ordinary, and the child gets a runtime +it never uses exposed while its own is absent from its filesystem view. That +failure arrives from inside the dynamic loader rather than as anything naming a +path, so a correctly pinned install fails every build for no visible reason. +Each recorded base is held to exactly the refusals the environment root is — +being read out of a file makes a path no narrower than guessing it would — and +an environment that records no base that still exists is refused with an +instruction rather than built against whatever runtime is lying around. + +The prefix a particular build ends up with is probed before that build runs, +not just the host's mechanism at startup: the readable set of a real build is +the provider's install rather than the interpreter paths the host probe uses, +and a widened exposure that reopened the boundary would otherwise meet nothing +between the exposure and the provider. The verification probe is handed this +build's exposure *plus* the interpreter, so that the probe child can start at +all; that makes the probed prefix strictly more permissive than the one the +build runs under, and containment observed there is a sound statement about +containment here. + +A `(deny default)` profile on macOS has to say one thing that is not about the +build's own exposure at all: Apple's `dyld-support.sb` is imported, because a +modern dynamic linker cannot reach the shared cache without it. The failure +without it is worth naming, since it is not a denied `open` — dyld aborts +inside `CacheFinder` before it owns `stderr`, so the child arrives as a +`SIGABRT` with no output of any kind and the profile reads as "this launcher +cannot start a child" on every macOS host. The import grants no general file +access. The other way to get dyld started, an unfiltered `(allow +file-read-data)`, would dissolve the boundary the profile exists to draw. + +No mechanism is trusted on its name, and none is looked up on `PATH`: each +candidate is an absolute path whose file and every ancestor directory must be +owned by root or by this user and unwritable by anyone else, because a launcher +somebody else can replace is a verdict somebody else can forge. + +A candidate is accepted only after a probe child launched behind it has been +*observed* failing at both halves: failing to reach a TCP listener this process +is really holding open on loopback, and failing to read a secret file planted +outside its exposure. The network verdict is taken at the listener, not from +the child's errno: a network namespace brings up its own loopback, so a +correctly contained child sees the same `ECONNREFUSED` that an unconfined child +sees from an unused host port. Those two are indistinguishable at the child and +obvious at the listener, which either accepted a connection or did not. A child +that fails one half and not the other is not a boundary; it classifies as +unusable. + +The child also prints a digest of a nonce generated for that run, so a launcher +that never started its child cannot pass for a boundary by exiting with the +contained code. + +An unsandboxed control child runs first and must reach the listener *and* read +the planted secret. If it cannot — no probe interpreter, loopback unavailable — +then "could not" proves nothing about any candidate, every candidate would +pass, and the probe refuses outright instead. The result is cached for the +process, since it is a property of the host. + +A host where no candidate passes gets no build. `subprocess_indexer()` raises +before a single blob is materialized, and `context-graph doctor` reports the +same condition as a failing `context-graph-isolation` check once a provider is +pinned. Running an unconfined provider is not offered as a fallback: an +operator who cannot contain a third-party indexer is better served by knowing +it than by a build that quietly could have reached the network. + +A Linux host that restricts unprivileged user namespaces — Ubuntu 24.04 and +GitHub's hosted runners among them — offers no mechanism by default. Installing +bubblewrap (`apt install bubblewrap`), which ships an AppArmor profile +permitting the namespaces it needs, is the least invasive way to give such a +host one, and is what the `graph containment` CI job does before running these +tests for real rather than skipping them. The alternative is to +lift the restriction system-wide +(`sysctl kernel.apparmor_restrict_unprivileged_userns=0`), which is a decision +about the whole machine rather than about this build, and not one this +repository makes on an operator's behalf. + +macOS has its own job, `graph containment (macOS)`, because the Linux job +proves the bubblewrap boundary and nothing whatever about the Seatbelt one, and +leaving the profile to be exercised only by whoever happened to run the suite +on a laptop is how it went unexecuted. `sandbox-exec` ships with the OS, so +there is nothing to install and the job is simply the evidence that the profile +runs at all. It does not gate merges yet: a red result there carries the +probe's own account of which candidate failed and what the launcher said on +stderr, which is the diagnosis the equivalent Linux failure was fixed from. + +Git itself runs with `GIT_CONFIG_NOSYSTEM`, `GIT_CONFIG_GLOBAL=/dev/null`, and +`GIT_CONFIG_SYSTEM=/dev/null`: an untrusted checkout's local, global, or system +configuration can otherwise install clean/smudge filters and hook paths that run +code during what looks like a read. + +Git children are not inside the provider's sandbox — they are children of Code +Mower itself — so the boundary has to reach them separately. Both invocation +paths, the census reader and the blob materializer, share one environment: +`GIT_NO_LAZY_FETCH=1`, and `GIT_ALLOW_PROTOCOL` set but empty, which git reads +as the complete list of permitted transports. `protocol.allow=never` travels on +the command line because that is the only level that outranks the repository's +own `.git/config`, which belongs to the untrusted checkout and is always read. + +The same environment carries `GIT_NO_REPLACE_OBJECTS=1`. A `refs/replace` entry +substitutes one object's bytes for another's on every ordinary read, so without +it a census and a materialization could bind content the commit and tree the +manifest records do not contain — and removing the replacement afterwards would +leave `status` still reporting `current`, because staleness is decided by +comparing object names. + +That still leaves the repository *shape* that makes a read reach out at all, so +**a build refuses a partial clone outright**. Where `extensions.partialclone` or +a promisor remote is configured, `ls-tree` and `cat-file` can fetch a missing +object from a remote mid-build. There is no bounded way to prove in advance +which objects are present locally, so the build declines the checkout rather +than discovering the gap one blob at a time. Use a full clone. + +## What a manifest binds + +Every published generation carries, in `manifest.json`: + +| Field | Why it is there | +| --- | --- | +| `commit`, `tree` | Full object names. Staleness is decided against these, not against a branch. | +| `provider` | Distribution, exact version, wheel SHA-256, and the extraction options used. The distribution and version are [checked against the install](#the-named-install-is-checked-against-the-pin-before-it-runs) before a build runs, so this is a record of what ran rather than of what was asked for. | +| `built_at` | ISO 8601 UTC. The provider records no build time of its own. | +| `tracked_files`, `tracked_bytes`, `census_digest` | Exactly which bytes the indexer was shown, re-derivable from the repository. | +| `skipped_paths` | How many tracked entries were deliberately not materialized. | +| `graph_digest`, `graph_bytes` | Detects a truncated or tampered artifact on every read. | +| `completeness` | `complete` or `partial`, from the provider's own admission. | +| `indexed_files` | What the provider claims it processed, bounded by the census. | + +`shareable_summary()` is the metadata-only view: revisions, digests, counts and +states. It carries no indexed content, no provider output, and no local path. + +## How the provider is actually invoked + +`subprocess_indexer()` builds the argv for the interface the adopt decision +evaluated, not a conventional-looking one: `extract` plus the pinned options, +run with its working directory set to the materialized copy. The evaluated +release takes no `--source`/`--output` pair — `extract` reads the directory it +is run in and writes its state beside those sources, which the clean-room run in +[the evaluation](graphify-evaluation.md) recorded as +`extract --code-only --no-cluster --max-workers 4`. + +**`--code-only` and `--no-cluster` are always passed, whatever the pin says.** +They are conditions of the adopt decision, not preferences: a pin that named no +options at all would otherwise have launched the provider into clustering and +whatever extraction it does by default, both of which are separate decisions +nobody has taken. They are folded into the pin's own `options` rather than added +at the launch site, so the manifest records the run that actually happened, and +a pin that tries to undo one of them — `--cluster`, `--no-code-only`, or a +valued form such as `--code-only=false` — is refused rather than quietly +overridden by argument order. + +So the adapter collects an artifact afterwards rather than naming one up front. +The state directory the provider wrote (`.graphify` or `.graph`, both already on +the excluded-roots list) is packed into a single reproducible archive: names +sorted, timestamps and ownership fixed, modes normalized, symlinks dropped. Two +builds of one commit have to produce identical bytes, because the manifest binds +a digest of them. That state lands inside the throwaway materialized copy, never +inside the indexed checkout, and the copy is deleted when the build ends. + +Packing is bounded as it happens, in both dimensions. The number of entries is +capped while their names are collected, and the serialized archive is written +into a buffer that refuses to grow past the artifact budget. Summing file sizes +is not a bound on the archive: many empty files stay far under the byte budget +while their headers, padding and extended pathname records are bytes this +process has to hold, and a budget checked on a finished archive is checked after +the memory was already taken. + +Extraction refuses to run at all over a state directory that already exists. +The census keeps committed provider state out of the materialized copy, so in a +build from this module there is none; the refusal is the second check, because +everything after the run treats whatever is in that directory as output this +run produced. + +**Completeness is read from the provider's report, never from its exit status, +and only an affirmative claim counts.** `complete` requires a report shaped the +way the adapter understands one: a claim that the run finished (`complete`, +`completed`, `finished`, or a recognized `status`), a count of what was +indexed, and no counter admitting requeued, pending, or failed work. Everything +else is `partial` — a report that denies completion, one in an unrecognized +schema, an empty object, an unreadable one, one larger than a manifest, and no +report at all. Absent evidence is not evidence of a complete build, and +`partial` is the state `graph_status` refuses by default, so the failure is one +an operator can see and act on. This is the direct consequence of the requeue +defect the evaluation recorded — a repeat that exits zero in 1.63 seconds +having requeued 54 entries has not built a complete graph. + +The report is provider output of unknown size, so it is read to one byte past +the manifest bound and refused if it is longer, rather than loaded whole and +measured afterwards. A bound checked on bytes already in memory bounds nothing. + +The provider's own stdout and stderr are the other unbounded output, and they +are discarded at the kernel: `stdin`, `stdout` and `stderr` are all +`DEVNULL`. Nothing reads them — completeness comes from the report, not from +what the run printed — so buffering them would only accumulate whatever a +talkative indexer chose to log, for up to the timeout, under neither the +tracked-content budget nor the artifact one. Inheriting them is not the +alternative: diagnostics can echo indexed source, and the process that launched +the build may be writing a machine-readable report to its own stdout. + +**Extraction runs in its own process group, and a run that overruns is stopped +as a group.** The adapter waits on the child itself rather than handing the run +to `subprocess.run`, whose timeout kills only the immediate child: an indexer +that started workers — and under a launcher such as `sandbox-exec` the direct +child is the launcher, not the indexer — would otherwise leave them running, +still holding CPU and still writing into a scratch directory the build deletes +as soon as it reports the failure. The group gets `SIGTERM`, a short grace +period, then `SIGKILL`, and the timeout is reported only once nothing is left +running. A new session is safe here precisely because no stream is inherited. + +The same check runs when the indexer simply exits, because its exit says nothing +about workers it started: one that is still writing would otherwise have its +output packed mid-write, and its scratch directory removed underneath it. The +group is looked up once at launch and kept — after the leader is reaped its pid +is no longer a safe thing to look a group up from — and an exit that left an +empty group behind costs one signal-`0` probe and no waiting at all. + +The subcommand, the state-directory names, and the report counters are constants +in one place in `context_graph_lifecycle.py`. They encode the interface as the +evaluation recorded it; the first installation against a real pinned release +should confirm them against that install and correct them here if they have +moved. + +## Refresh is explicit + +`build` is the first-time verb and refuses when a usable generation already +binds the revision. `refresh` is the rebuild verb, and it publishes a *new* +immutable generation rather than mutating one in place. Nothing refreshes on a +timer, a hook, or a file-system event, and nothing rebuilds implicitly because a +consumer found the graph stale — a stale graph is reported as stale. + +## Failing closed + +`graph_status()` resolves to exactly one state, and only `current` is usable. +Nothing falls back to an older generation: a consumer that cannot have the +revision it asked for is told so rather than handed a stale answer that looks +fresh. + +| State | Cause | +| --- | --- | +| `absent` | Nothing built for this checkout. | +| `stale` | The manifest's commit/tree does not match the revision being asked about. | +| `corrupt` | The artifact's size or SHA-256 does not match the manifest. | +| `oversized` | The artifact exceeds its budget. | +| `partial` | The provider declared an incomplete build. Usable only with an explicit opt-in. | +| `invalid` | The manifest is unreadable, mislabelled, or the state is not private and operator-owned. | + +The privacy check runs on every read, not only at creation: state loosened after +the fact — by a umask change, a restore, or a careless recursive `chmod` — +fails closed rather than being trusted because it was private when it was +written. + +`partial` exists because of the requeue defect recorded in the evaluation: a +fast incremental repeat is not proof that the graph is complete. + +## Commands + +``` +code-mower context-graph build --pin-file PIN --indexer PATH [--revision REV] +code-mower context-graph refresh --pin-file PIN --indexer PATH [--revision REV] +code-mower context-graph status [--allow-partial] [--json] +code-mower context-graph remove [--show-local-paths] +code-mower context-graph doctor [--pin-file PIN] +``` + +`status` exits non-zero when the graph is not usable, so a script can branch on +it. `build` and `refresh` do the same, and for the same reason: a provider that +admitted an incomplete run has published a generation `status` will call +`partial` and refuse, so the build prints `partial` and exits non-zero rather +than describing it as `current` for as long as it takes to ask again. +`doctor` reports `skip` rather than `fail` when nothing is pinned or built: +the lifecycle is optional, and an operator who never opted in has nothing wrong +with their installation. + +The pin file names one exact release and is rejected if it names a range, a +marker, or a distribution without an artifact digest: + +```json +{ + "distribution": "graphifyy", + "version": "0.9.58", + "wheel_sha256": "e239803288e91c723d6e30540860bd6d5a1dc3f0914b9fc1104b0233e98aaeb8", + "options": ["--code-only", "--no-cluster"] +} +``` + +`options` is optional and may name extra provider flags, such as +`--max-workers`. The two restrictions above are added whether or not the file +lists them; listing them changes nothing, and contradicting them is refused. +The list is bounded, and the bound is counted on the options as they will +actually run — the two required flags included — so a pin that passes validation +always round-trips through the manifest it is recorded in. + +`--indexer` is the path to a provider CLI the operator has **already** +installed. This repository does not download, install, or resolve one, which is +why the executable is named rather than discovered. Whatever is named is bound +to one absolute path, decided in the directory the command was invoked from: a +relative path such as `.venv/bin/graphify` is resolved against that directory +rather than against the materialized copy the provider runs in, and a bare +command name is looked up on `PATH` there, once. `PATH` is not independent of +the child's directory — an entry on it can itself be relative, and +`PATH=provider-venv/bin` names a different directory once the child starts in +the materialized copy. Leaving the lookup to the launch meant containment was +drawn around the install this process found while the child searched somewhere +else, so a correctly installed provider failed to start; and a repository +carrying that same relative path would have answered the child's search with a +tracked file, which is a build executing content it was only ever meant to read. + +### The named install is checked against the pin, before it runs + +`--indexer` names an install and `--pin-file` names a release, and nothing used +to compare the two. The manifest records the pin as the provenance of every byte +in a generation, so a build could publish a manifest naming graphifyy 0.9.58 +over a graph that some other release — or some other distribution that happens +to answer to `extract` — had produced, and neither `status` nor the manifest +could tell afterwards. For a distribution whose name differs from this +repository's by one character, that is precisely the substitution the pin exists +to make identifiable. + +So the executable is checked against the pin before anything is materialized: + +- **Which distribution installed it.** Ownership is read from the installed + `RECORD`, not guessed from the file's name, because a pinned release and a + lookalike can both ship a console script called `graphify` and an environment + is free to hold both. An executable no installed distribution claims is + refused. +- **At which version.** `Name` and `Version` come from that distribution's + `METADATA`. Names are compared the way an installer normalizes them (PEP 503), + so a pin does not fail against its own install over a spelling; versions are + compared exactly, because a pin is one release and deciding that `1.0` and + `1.0.0` are the same release is a policy this has no business inventing. +- **Whether the file is still what was installed.** The executable is hashed + against the digest its installer recorded for it, so an install rewritten in + place is refused even though its `.dist-info` still names the pinned release. + An entry whose digest the installer left empty is not treated as a mismatch. + +The install is *read*, never *asked*. Running `graphify --version` would mean +launching the very executable whose identity is in question, outside the sandbox +that exists to confine it, and then believing what it said about itself. + +`wheel_sha256` stays unchecked here and is not checkable here: an unpacked +install does not retain the artifact it came from. It remains the operator's +record of which artifact they installed, carried into the manifest so a +substituted distribution is identifiable after the fact. + +## State layout + +``` +~/.local/share/code-mower/context/graph// + current the published generation's name + build.lock serializes builds for one checkout + generations//manifest.json + generations//graph.bin +``` + +Directories are 0700 and files 0600. `` is derived from the +checkout's **worktree root**, asked of Git rather than taken from the directory +the command was run in. A census reads the commit's whole tree, so `status` in +`src/` asks about exactly the generation `build` at the root published and has +to resolve to it; deriving the name from the invocation directory made every +subdirectory its own workspace, and a graph built at the root then read as +`absent` from `src/` while `remove` there deleted nothing and reported success. +Git answers per worktree, so two worktrees of one repository still get separate +state and can never read each other's generations — each may hold a different +revision. A path Git cannot place — not a repository, a bare one, or no Git on +the host — keeps its resolved path, so state stays nameable and the verbs that +need Git fail on their own terms. The same root is what the provider exposure +rule is drawn against, so a provider installed inside the checkout is refused +from a subdirectory exactly as it is from the root. State is refused inside any Git +repository, which is the enforcement half of adoption condition 2. The refusal +is checked on the resolved path as well as the given one: `--state-dir +/outside/link/state` names no repository in its own spelling while +`/outside/link` points inside one. Symlinked ancestors are resolved rather than +rejected — ordinary private roots have them, macOS reaches `/tmp` through a +link into its `private` directory. + +Resolving settles what the ancestors mean at construction and nothing about +what they become afterwards, so the root is opened by walking it from `/` one +component at a time, each against its parent's descriptor with `O_NOFOLLOW`. +Opening the whole absolute path in one call would not do: `O_NOFOLLOW` refuses +only the *final* component, and every ancestor above it is resolved exactly as +a link planted there would want. Nor does finding the deepest existing prefix +first — that prefix is still opened by its full spelling. A pre-created root +behind a newly inserted ancestor link therefore used to be accepted, because +the leaf really is a directory and really is not a link; it is simply not the +directory that was checked. + +Renames and removals still travel full paths — a staged generation is renamed +into place, a removed tree is recursed over — and a full path is re-resolved +from the root on every call. Before each of those, the inode the no-follow walk +arrived at is compared against the one the path spells now, and a mismatch is a +refusal rather than a write into whatever the link points at. + +## What this does not do + +No hooks, no watcher, no hosted service, no MCP HTTP service, no semantic or +model-based extraction, no provider API key, no clustering, and no default +dependency. Each remains a separate explicit decision. The provider seam is an +injected callable, so the entire lifecycle — including the whole test suite — +runs offline with no graph package installed. diff --git a/docs/context-provider-contract.md b/docs/context-provider-contract.md index 7c1590e2..d6252fe1 100644 --- a/docs/context-provider-contract.md +++ b/docs/context-provider-contract.md @@ -205,3 +205,11 @@ package record, and the conditions an implementing change must meet. Nothing is installed or required yet. Synthetic graph fixtures prove only the extension point; they do not establish Graphify compatibility or make it a v1.3.1 dependency. + +The lifecycle around such a provider — exact pin, immutable tracked-file +materialization, scrubbed environment, an OS sandbox that denies the provider +the network, private 0700 state, atomic generations, +and `code-mower context-graph build/refresh/status/remove/doctor` — is +described in [Local repository graph](context-graph-lifecycle.md). It builds the +evidence side; `context_graph` still decides whether a delivered packet's +citations may be used. diff --git a/docs/current-state-and-roadmap.md b/docs/current-state-and-roadmap.md index 650fd611..0d632421 100644 --- a/docs/current-state-and-roadmap.md +++ b/docs/current-state-and-roadmap.md @@ -158,7 +158,10 @@ participant. Start local and code-only: [issue #876](https://github.com/codemower-ai/code-mower/issues/876) with an adopt decision; - add a provider registry and multiple context attachments per session; -- build and refresh graphs with commit/freshness validation; +- build and refresh graphs with commit/freshness validation — delivered by + `code-mower context-graph`, described in the + [lifecycle record](context-graph-lifecycle.md), which closes + [issue #913](https://github.com/codemower-ai/code-mower/issues/913); - consume a pinned structured JSON contract; - generate bounded impact, dependency, symbol, and related-test packets; and - deliver the same packet shape to Claude, Codex, and Devin. diff --git a/docs/graphify-evaluation.md b/docs/graphify-evaluation.md index 66c09291..22af78fc 100644 --- a/docs/graphify-evaluation.md +++ b/docs/graphify-evaluation.md @@ -167,6 +167,16 @@ are engineering conditions on the adapter, not requests for a decision. accounted for: an incremental run's completion is not treated as proof the graph is complete. +Conditions 1, 2, 3, 5 and 7 are implemented by the build/refresh/status/remove +lifecycle in +[Local repository graph: revision-bound lifecycle](context-graph-lifecycle.md) +(issue #913): an exact pin with a verified artifact digest, private 0700 state +outside every checkout, a manifest that binds full commit and tree with build +time, opt-in acquisition with no default dependency, and a `partial` +completeness state that refuses to read a fast incremental repeat as a complete +graph. Conditions 4 and 6 belong to the retrieval adapter, which does not exist +yet. + ## Boundary Graphify stays out of v1.3.1 and does not block Coworker's 1.3.0 or 1.3.1 diff --git a/src/code_mower/cli.py b/src/code_mower/cli.py index d30500b7..0b9117ee 100644 --- a/src/code_mower/cli.py +++ b/src/code_mower/cli.py @@ -56,6 +56,7 @@ def _source_checkout_install_spec() -> str: from . import controller as code_mower_controller from . import code_mower_calibration from . import code_mower_context_packs +from . import context_graph_command from . import code_mower_merge from . import code_mower_telemetry from . import config as code_mower_config @@ -447,6 +448,7 @@ def _local_llm_main(argv: list[str]) -> int: "cloud": "Export or upload sanitized benchmark metadata.", "config": "Validate or inspect a Code Mower config.", "context": "Record local external planning context manifests.", + "context-graph": "Build, refresh, inspect, or remove a local repository graph.", "context-packs": "Build selective surrounding-file context packs.", "controller": "Compute supervised-pilot dispatch and merge-policy decisions.", "coderabbit-cli": "Run a CodeRabbit CLI informational lane.", @@ -595,6 +597,7 @@ def _top_level_help(show_all: bool) -> str: "cloud": code_mower_cloud.main, "config": _config_main, "context": code_mower_work_orders.context_main, + "context-graph": context_graph_command.main, "context-packs": code_mower_context_packs.main, "controller": code_mower_controller.main, "coderabbit-cli": coderabbit_cli_audit_pr.main, diff --git a/src/code_mower/context_graph_command.py b/src/code_mower/context_graph_command.py new file mode 100644 index 00000000..a78108f7 --- /dev/null +++ b/src/code_mower/context_graph_command.py @@ -0,0 +1,167 @@ +"""``code-mower context-graph``: build, refresh, inspect and remove a local graph. + +The lifecycle in ``context_graph_lifecycle`` is deliberately not wired into any +default path. This command is how an operator opts in, one checkout at a time, +and it asks for everything explicitly rather than discovering it: the provider +pin comes from a file the operator names, and the indexer executable comes from +an install the operator already made. Nothing here downloads, installs, or +resolves a provider. + +Output is metadata only -- revisions, digests, counts, and states. No indexed +content, provider output, or local path of the private state directory is +printed unless the operator asks for it with ``--show-local-paths``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from . import context_graph_lifecycle as lifecycle +from .context_contract import ContextError +from .context_store import strict_json + +MAX_PIN_BYTES = 8192 + + +def _load_pin(path: Path | None) -> lifecycle.GraphifyPin | None: + if path is None: + return None + try: + # Bounded at the stream, not after the fact: a bound checked on bytes + # already in memory is not a bound on what the file can cost to read. + with path.open("rb") as stream: + raw = stream.read(MAX_PIN_BYTES + 1) + except OSError: + raise ContextError("local graph provider pin file is unreadable") from None + if len(raw) > MAX_PIN_BYTES: + raise ContextError("local graph provider pin file exceeds its bound") + return lifecycle.load_pin(strict_json(raw)) + + +def _require_pin(path: Path | None) -> lifecycle.GraphifyPin: + pin = _load_pin(path) + if pin is None: + raise ContextError("building a local graph requires an exact provider pin") + return pin + + +def _emit(payload: dict, *, as_json: bool, text: str) -> None: + if as_json: + print(json.dumps(payload, indent=2, sort_keys=True)) + else: + print(text, end="") + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="code-mower context-graph", + description="Manage an optional revision-bound local repository graph.", + ) + sub = parser.add_subparsers(dest="command", required=True) + build = sub.add_parser("build", help="Build and publish a generation for the current revision") + refresh = sub.add_parser("refresh", help="Explicitly rebuild and atomically publish a new generation") + status = sub.add_parser("status", help="Report whether the published generation may be used") + remove = sub.add_parser("remove", help="Delete this checkout's private local graph state") + doctor = sub.add_parser("doctor", help="Check the local graph posture without building anything") + + for command in (build, refresh, status, remove, doctor): + command.add_argument("--repo-path", type=Path, default=Path.cwd(), help="Checkout to bind") + command.add_argument("--state-dir", type=Path, help="Private state root; defaults to the context store") + command.add_argument("--json", action="store_true", help="Emit a machine-readable summary") + for command in (build, refresh, status, doctor): + command.add_argument("--revision", default="HEAD", help="Revision to bind, for example a commit or tag") + for command in (build, refresh, doctor): + command.add_argument("--pin-file", type=Path, help="JSON file naming one exact provider release") + for command in (build, refresh): + command.add_argument("--indexer", required=True, help="Path to the already-installed pinned provider CLI") + command.add_argument("--keep-previous", action="store_true", + help="Retain superseded generations instead of pruning them") + status.add_argument("--allow-partial", action="store_true", + help="Treat a provider-declared partial build as usable") + remove.add_argument("--show-local-paths", action="store_true", help="Include the private state path in output") + + args = parser.parse_args(argv) + try: + if args.command in ("build", "refresh"): + pin = _require_pin(args.pin_file) + if args.command == "build" and lifecycle.graph_status( + args.repo_path, root=args.state_dir, revision=args.revision + ).usable: + # ``build`` is the first-time verb. A usable generation already + # binds this revision, so rebuilding it is ``refresh`` -- an + # explicit choice, never something ``build`` does by surprise. + raise ContextError("a current generation already binds this revision; use refresh to rebuild") + manifest = lifecycle.build_graph( + args.repo_path, + pin=pin, + # The pin goes to the adapter as well as to the build: the + # adapter checks the install it is about to run against it, so + # a manifest never records a release nobody confirmed was + # installed. + indexer=lifecycle.subprocess_indexer( + args.indexer, repository=args.repo_path, pin=pin + ), + root=args.state_dir, + revision=args.revision, + keep_previous=args.keep_previous, + ) + # The published state is the manifest's, not this command's to + # assume: a provider that admitted an incomplete run has published + # a generation ``status`` will call ``partial`` and refuse, and + # printing ``current`` here would describe it as usable for exactly + # as long as it took the operator to ask again. + complete = manifest.completeness == lifecycle.COMPLETE + published = lifecycle.GenerationStatus( + state="current" if complete else "partial", + generation=manifest.generation, + manifest=manifest, + detail="" if complete else "local graph build was incomplete; refresh it", + ) + summary = {"status": "published", "usable": published.usable, **manifest.shareable_summary()} + _emit(summary, as_json=args.json, text=lifecycle.render_status_text(published)) + # Publishing an unusable generation is a reportable condition, not + # a crash: exit non-zero for the same reason ``status`` does, so a + # script does not have to re-ask to find out what it just built. + return 0 if published.usable else 1 + if args.command == "status": + report = lifecycle.graph_status( + args.repo_path, + root=args.state_dir, + revision=args.revision, + require_complete=not args.allow_partial, + ) + _emit(report.shareable_summary(), as_json=args.json, text=lifecycle.render_status_text(report)) + # A non-current graph is a normal, reportable condition, not a + # command failure; exit 1 so a script can branch on usability. + return 0 if report.usable else 1 + if args.command == "remove": + state = lifecycle.GraphStateRoot(args.repo_path, root=args.state_dir) + path = str(state.path) if args.show_local_paths else None + removed = lifecycle.remove_graph(args.repo_path, root=args.state_dir) + payload = {"schema": "code_mower.contextGraphRemove.v1", "removed": removed} + if path is not None: + payload["path"] = path + _emit(payload, as_json=args.json, + text=("Removed local graph state.\n" if removed else "No local graph state to remove.\n")) + return 0 + report = lifecycle.doctor_report( + args.repo_path, pin=_load_pin(args.pin_file), root=args.state_dir, revision=args.revision + ) + lines = [f"Local graph doctor: {report['status']}"] + lines.extend(f" [{check['status']}] {check['check']}: {check['message']}" for check in report["checks"]) + _emit(report, as_json=args.json, text="\n".join(lines) + "\n") + return 0 if report["status"] != "fail" else 1 + except ContextError as error: + print(f"local graph unavailable: {error}", file=sys.stderr) + return 1 + except Exception: + # Never let a provider or filesystem failure surface indexed content. + print("local graph unavailable; verify the pin, the checkout and the private state directory", file=sys.stderr) + return 1 + + +if __name__ == "__main__": # pragma: no cover - direct invocation + raise SystemExit(main()) diff --git a/src/code_mower/context_graph_lifecycle.py b/src/code_mower/context_graph_lifecycle.py new file mode 100644 index 00000000..cb77bda7 --- /dev/null +++ b/src/code_mower/context_graph_lifecycle.py @@ -0,0 +1,3524 @@ +"""Revision-bound lifecycle for an optional local-repository graph (issue #913). + +``context_graph`` decides whether a graph's *citations* are in scope. This +module decides whether the graph should have existed at all: which revision it +binds, which bytes it was allowed to see, where its state lives, and when a +consumer must refuse it. + +The rules a local indexer cannot be trusted to follow on its own: + +* **Never index a live checkout.** A working tree mutates mid-build and carries + untracked, ignored, and private files. Every build materializes the tracked + blobs of one commit into a private staging directory and points the indexer + at that copy instead. Untracked and ignored files have no path into the + graph because they are never written. +* **Bind the revision, not the branch.** An artifact records the full commit + and tree SHA it was built from. A consumer compares those against the + repository it is actually asking about; a mismatch is stale, and stale fails + closed rather than answering from the wrong revision. +* **Publish atomically, immutably.** A generation is assembled under a staging + name, fsynced, then renamed into place; the ``current`` pointer is replaced + atomically afterwards. A reader either sees the whole previous generation or + the whole new one, never a half-written directory. +* **Check the install against the pin.** A manifest records the pin as the + provenance of every byte in a generation, so the executable a build is about + to run is checked against it first: which distribution installed it, at which + version, and whether it is still the file that installer wrote. The install + is read for this rather than the provider asked, because asking would mean + running the executable whose identity is in question. +* **Scrub the environment.** The indexer runs with an allowlisted environment, + so an ambient token cannot leak into a provider process. +* **Deny the network and the host filesystem in the kernel, not by request.** + Emptying proxy variables only redirects a client that chooses to honour them, + and a working directory is not a boundary. The provider is launched inside an + OS sandbox whose filesystem view is the materialized copy, the build's own + scratch directories, and a read-only runtime -- the operator's home, their + other checkouts, and every ignored ``.env`` beside them are absent from it, + not merely unreadable. The mechanism is accepted only after a probe child has + been observed failing at *both*: failing to reach a socket this process is + really listening on, observed at the listener rather than believed from the + child's errno, and failing to read a secret file planted outside its + exposure. A host that offers no such mechanism gets a refused build, not an + unconfined provider. + +Nothing here installs, imports, or requires a graph package. The indexer is an +injected callable, so the whole lifecycle is provable offline; the bundled +``subprocess_indexer`` builds the argv and the scrubbed environment for a +pinned provider without this module depending on it. +""" + +from __future__ import annotations + +import base64 +import contextlib +import csv +import hashlib +import json +import os +import re +import io +import secrets +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence + +from .context_contract import ContextError, _identifier, _text, _timestamp +from .context_graph import _EXCLUDED_ROOTS +from .context_store import _private, default_context_root +from .file_locks import FileLockError, exclusive_handle_lock + +MANIFEST_SCHEMA = "code_mower.contextGraphBuild.v1" +MANIFEST_NAME = "manifest.json" +ARTIFACT_NAME = "graph.bin" +CURRENT_NAME = "current" + +#: Bounds. A local graph is a convenience, not a reason to fill a disk or to +#: stall a session on a pathological repository. Every one of these fails the +#: build closed rather than truncating silently. +MAX_MANIFEST_BYTES = 262_144 +MAX_ARTIFACT_BYTES = 256 * 1024 * 1024 +MAX_TRACKED_FILES = 50_000 +MAX_TRACKED_BYTES = 512 * 1024 * 1024 +MAX_BLOB_BYTES = 32 * 1024 * 1024 +#: How many files the provider's state may contribute to one artifact. A byte +#: budget alone is not a bound on what packing costs: many empty files stay far +#: under it while their headers, padding and extended pathname records grow the +#: archive this process holds. Sized to the tracked-file bound, because a graph +#: of a checkout has no honest reason to hold more entries than the checkout +#: had files. +MAX_ARTIFACT_ENTRIES = MAX_TRACKED_FILES +#: How many skipped paths one census may record. The file-count budget above +#: bounds only what is materialized, so a repository of symlinks, submodules, or +#: committed provider state passes it while the skipped list grows without +#: limit. A manifest's ``skipped_paths`` is validated against this bound on +#: every read, so a census that could exceed it would build a generation that +#: publishes, prunes its predecessor, and then reads back ``invalid``. Bounded +#: where the entries are collected, before any of that happens. +MAX_SKIPPED_PATHS = MAX_TRACKED_FILES + +#: Only ordinary blobs are materialized. A symlink (``120000``) can name a +#: target outside the checkout and a gitlink (``160000``) names a commit in +#: another repository this build was never authorized to read. +_REGULAR_MODES = frozenset({"100644", "100755"}) +_SKIPPED_MODES = {"120000": "symlink", "160000": "submodule"} + +#: Where the provider keeps its own index state. Both names are on the +#: excluded-roots list in ``context_graph``, and a repository is free to track +#: either of them -- a committed ``.graph/`` is somebody else's graph, or an +#: earlier incremental cache of this one. Neither may be materialized: the +#: provider would then resume from a cache built over content this build never +#: saw, and the adapter would collect tracked repository bytes as if the +#: provider had just produced them, binding stale contents to a fresh commit. +#: Matched at any depth and case-folded, for the same reasons ``.git`` is. +_PROVIDER_STATE_DIRECTORIES = (".graphify", ".graph") +_PROVIDER_STATE_ROOTS = frozenset(name.casefold() for name in _PROVIDER_STATE_DIRECTORIES) + +#: The evidence contract's excluded roots, bound rather than copied. One module +#: decides a citation into private state is out of scope, this one decides +#: those bytes never reach the indexer at all; they are the same policy read +#: from two ends, and a name added to one must not have to be remembered in the +#: other. The set is a superset of the provider roots above: it also carries +#: ``.git``, whose contents are the history rather than the revision, and +#: ``.code-mower``, this tool's own state. A repository is free to track +#: either, and a build over a tracked ``.code-mower/`` would hand the provider +#: exactly the packets and evidence that ``context_graph`` then refuses to let +#: a packet cite. +_PRIVATE_STATE_ROOTS = _EXCLUDED_ROOTS + +_OBJECT_NAME = re.compile(r"[0-9a-f]{40}(?:[0-9a-f]{24})?\Z") +_GENERATION = re.compile(r"[0-9a-f]{32}\Z") +_VERSION = re.compile(r"[0-9][0-9A-Za-z.+!-]{0,63}\Z") +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") + +COMPLETE = "complete" +PARTIAL = "partial" + +#: The only variables a provider process inherits. Everything else -- every +#: token, cloud credential, proxy, and provider API key in the operator's +#: session -- is dropped rather than filtered, so a newly invented secret +#: variable is excluded by default instead of needing a new denylist entry. +_ENVIRONMENT_ALLOWLIST = ("PATH", "TMPDIR", "LANG", "LC_ALL", "TZ") + +#: Hygiene, not the boundary. Emptying proxy variables stops a cooperating +#: client from finding a proxy and ``GIT_TERMINAL_PROMPT=0`` stops a child +#: blocking on a credential prompt, but on a host with direct connectivity +#: neither denies anything. The boundary is ``containment_prefix``. +_NETWORK_DENY = { + "no_proxy": "*", + "NO_PROXY": "*", + "http_proxy": "", + "https_proxy": "", + "HTTP_PROXY": "", + "HTTPS_PROXY": "", + "ALL_PROXY": "", + "all_proxy": "", + "GIT_TERMINAL_PROMPT": "0", + "PYTHONNOUSERSITE": "1", +} + +#: Isolation mechanisms, most specific first, each named by an absolute path. +#: +#: Absolute, deliberately: a launcher looked up on an inherited ``PATH`` can be +#: shadowed by a program that answers the probe without confining anything, and +#: the probe's verdict is only as good as its knowledge of what it ran. The +#: paths are the system locations these tools install into, and each is checked +#: for trusted ownership and unwritable ancestry before it is run. +#: +#: ``unshare --net`` used to be here and is gone. It denies the network and +#: nothing else: the child keeps the host's whole filesystem, which is not the +#: boundary this module claims. A host with no mechanism that confines *both* +#: gets a refused build. +_SANDBOX_CANDIDATES: tuple[tuple[str, str], ...] = ( + ("sandbox-exec", "/usr/bin/sandbox-exec"), + ("bwrap", "/usr/bin/bwrap"), + ("bwrap", "/usr/local/bin/bwrap"), +) + +#: Read-only host paths a runtime needs to start at all: the loader, the C +#: library, the system interpreters. Everything outside this list and the +#: exposure a build asks for is not in the child's filesystem view -- not +#: unreadable by permission, absent. +#: +#: Named one runtime directory at a time rather than one top-level directory at +#: a time. This list used to say ``/usr``, ``/etc`` and ``/Library``, which is a +#: far larger claim than "what a runtime needs to start": ``/usr`` carries +#: ``/usr/local`` -- Homebrew's whole prefix, its ``etc`` and ``var`` included -- +#: and ``/usr/src``, either of which can hold a checkout; ``/etc`` carries +#: whatever service credentials the host's packages left world-readable; and +#: ``/Library`` carries ``Keychains``, ``Preferences``, ``Application Support`` +#: and the rest of a Mac's machine-wide operator data. Every one of those was +#: added to every build unconditionally, so the refusals that guard an exposure +#: root never saw them, and a checkout under one of them stayed readable beside +#: the materialized copy that exists to replace it. +#: +#: The narrowing is deliberately fail-closed: a host whose runtime needs +#: something not named here refuses builds (the probe cannot start a child, so +#: no mechanism is verified) instead of running a provider with more exposed +#: than this list admits to. +_SYSTEM_READ_PATHS: tuple[str, ...] = ( + # The dynamic loader and the C library, in every spelling a distribution + # uses. ``/lib64`` and friends are how an ELF binary names its program + # interpreter even where they are links into ``/usr``. + "/lib", + "/lib64", + "/lib32", + "/usr/lib", + "/usr/lib64", + "/usr/lib32", + "/usr/libexec", + # System executables a runtime execs or reads: the system interpreters + # themselves live here. + "/bin", + "/sbin", + "/usr/bin", + "/usr/sbin", + # Architecture-independent runtime data: locales, time zones, ICU. Package + # data rather than operator data, and the runtime reads it during start-up. + "/usr/share", + # Apple's signed system volume, which no operator writes and no checkout + # can live on, plus the dyld cache's and time zone database's own state. + "/System", + "/private/var/db/dyld", + "/private/var/db/timezone", + # The two places on a Mac that hold a *runtime* rather than operator data: + # a python.org interpreter installs itself into ``/Library/Frameworks``, and + # Apple's command line tools keep their interpreter and its shared + # libraries inside their own bundle. Nothing else under ``/Library`` is a + # runtime dependency, and the rest of it is exactly the machine-wide data + # this boundary exists to keep away from a provider. + "/Library/Frameworks", + "/Library/Developer/CommandLineTools/usr/lib", + "/Library/Developer/CommandLineTools/Library/Frameworks", + # ``/etc``, entry by entry. The loader's cache and configuration, the time + # zone, the account databases a runtime resolves a home directory through, + # and OpenSSL's configuration file -- which its own providers read on + # initialization. Not ``/etc/ssl/private``, not a service's credentials, + # not whatever else a host keeps here. + "/etc/ld.so.cache", + "/etc/ld.so.conf", + "/etc/ld.so.conf.d", + "/etc/alternatives", + "/etc/localtime", + "/etc/timezone", + "/etc/passwd", + "/etc/group", + "/etc/nsswitch.conf", + "/etc/os-release", + "/etc/ssl/openssl.cnf", +) + +#: The probe reports two facts about one run, as a bitmask offset from a base +#: no shell error code lands on: whether the child could read a secret file +#: planted outside its exposure, and whether it could open a socket to a port +#: this process is really listening on. +#: +#: The network verdict is taken at the *listener*, not from the child's errno. +#: Classifying by errno cannot work: a network namespace brings its own +#: loopback up, so a contained child gets ``ECONNREFUSED`` from an empty +#: namespace while an unconfined child gets ``ECONNREFUSED`` from an unused host +#: port. Indistinguishable at the child; obvious at the listener, which either +#: accepted a connection or did not. +#: +#: An exit code alone is not evidence that the child ran: a launcher that exits +#: with the contained code without executing anything would be accepted as a +#: boundary while confining nothing. So the child first prints a value only +#: running it can produce -- the digest of a nonce generated for this run -- and +#: a run with no such evidence is unusable whatever its exit code. Echoing the +#: argv is not enough: the digest is computed by the child and the nonce is +#: fresh, so neither a launcher that parrots its arguments nor one that replays +#: an earlier probe can produce it. +_PROBE_BASE = 40 +_PROBE_READ_SECRET = 1 +_PROBE_REACHED_LISTENER = 2 +_CONTAINMENT_PROBE = """ +import hashlib +import socket +import sys + +print(hashlib.sha256(sys.argv[3].encode()).hexdigest(), flush=True) +seen = 0 +try: + with open(sys.argv[2], "rb") as secret: + secret.read(1) +except OSError: + pass +else: + seen |= 1 +try: + reached = socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=5) +except OSError: + pass +else: + reached.close() + seen |= 2 +sys.exit(40 + seen) +""" + +#: How a probe run classifies: the child was outside the boundary in at least +#: one respect, the child was inside it in both, or nothing usable happened. +#: A mechanism that denies only one of the two is ``_UNUSABLE``, not a +#: boundary -- half a boundary is what this finding was about. +_REACHED = "reached" +_CONTAINED = "contained" +_UNUSABLE = "unusable" + + +@dataclass(frozen=True) +class Containment: + """One verified isolation mechanism on this host. + + The argv is built per build rather than cached, because the boundary is a + function of what that build is allowed to expose. What is cached is the + finding that this mechanism, at this path, was observed confining a child. + """ + + name: str + launcher: str + + +_containment: Containment | None = None +_containment_probed = False + + +def _trusted_launcher(path: str) -> str | None: + """A launcher only a trusted account could have replaced, or ``None``. + + The file and every ancestor directory: an executable that is itself + root-owned but sits in a directory somebody else may write can be swapped + for one that reports containment it never established. A symlink anywhere + in the chain is refused rather than followed -- what it names now is not + what it will name later, and this decision is cached for the process. + """ + trusted = {0, os.geteuid()} + for current in (Path(path), *Path(path).parents): + try: + entry = os.lstat(current) + except OSError: + return None + if entry.st_uid not in trusted or entry.st_mode & (stat.S_IWGRP | stat.S_IWOTH): + return None + if stat.S_ISLNK(entry.st_mode): + return None + info = os.lstat(path) + if not stat.S_ISREG(info.st_mode) or not os.access(path, os.X_OK): + return None + return path + + +def _existing(paths: Iterable[Path | str], *, follow: bool = True) -> tuple[str, ...]: + """De-duplicated existing paths, in the order they were given. + + ``follow`` is which spelling the mechanism decides on. A seatbelt + ``subpath`` matches the kernel's resolved path -- macOS puts ``/tmp`` and + ``/var`` behind links into its ``private`` directory -- so an unresolved + exposure there would name a path the sandbox never sees. + + A bind mount is the other way round. The destination is a literal path in + an otherwise empty root, and Linux's ``/lib64 -> usr/lib64`` compatibility + links are how every ELF binary names its program interpreter. A root with + ``/usr/lib64`` and no ``/lib64`` cannot exec anything at all, and says so + as an ``execvp`` ENOENT naming the binary rather than the loader it could + not find. So a mechanism that does not follow gets both spellings. + """ + seen: dict[str, None] = {} + for path in paths: + literal = os.fspath(path) + try: + real = os.path.realpath(literal) + except OSError: # pragma: no cover - realpath does not raise on absent paths + continue + for candidate in (real,) if follow else (literal, real): + if os.path.exists(candidate): + seen.setdefault(candidate, None) + return tuple(seen) + + +def _seatbelt_literal(path: str) -> str: + return '"' + path.replace("\\", "\\\\").replace('"', '\\"') + '"' + + +#: Apple's own bootstrap rules for the dynamic linker. On a ``(deny default)`` +#: profile a modern dyld cannot reach the shared cache, and it does not fail as +#: a denied open: it aborts inside ``dyld4::CacheFinder`` before it owns stderr, +#: so the child arrives as ``SIGABRT`` with no stdout and no stderr and the +#: probe can only report that the launcher started nothing. The rules it needs +#: are the cryptex cache paths plus the narrow ``syscall-unix``, +#: ``system-fcntl`` and ``system-mac-syscall`` operations -- exactly what this +#: profile ships. Importing it grants no general file access; the alternative +#: that also starts dyld, an unfiltered ``(allow file-read-data)``, would +#: destroy the filesystem boundary this profile exists to draw. +_DYLD_SUPPORT_PROFILE = "/System/Library/Sandbox/Profiles/dyld-support.sb" + + +def _seatbelt_prefix(launcher: str, *, writable: Sequence[str], readable: Sequence[str]) -> tuple[str, ...]: + """A ``sandbox-exec`` profile that denies by default and then names the exposure. + + ``(allow default)(deny network*)`` -- what this module used to pass -- denies + sockets and leaves the host filesystem wide open. The order here is the + other way round: nothing is permitted, and then the runtime is made readable + and the build's own directories writable. + """ + rules = ["(version 1)"] + if os.path.exists(_DYLD_SUPPORT_PROFILE): + # Directly after ``(version 1)`` and before everything else: Apple's + # profile declares version 3, so an import ahead of this file's own + # version declaration will not compile. ``(deny default)`` follows it + # and remains the default posture -- a default is not a rule that + # overrides the import, which is why this placement is the one + # observed to both start dyld and keep the boundary. A host old enough + # not to ship the profile keeps the previous rules rather than failing + # to compile an import of a file that is not there; if its dyld needs + # them anyway, that host refuses the build instead of running it + # unconfined. + rules.append('(import "dyld-support.sb")') + rules += [ + "(deny default)", + "(deny network*)", + "(allow process-fork)", + "(allow signal)", + "(allow sysctl-read)", + "(allow mach-lookup)", + "(allow ipc-posix-shm)", + "(allow file-read-metadata)", + # Read *and* write on the same four devices. Granting write without + # read is an asymmetry nothing wants: a runtime opens ``/dev/null`` + # read-write to detach a stream, and reads ``/dev/urandom`` to seed + # itself, so a profile that denies the read denies the process its + # start rather than denying it anything an operator cares about. + "(allow file-read-data file-write-data (literal \"/dev/null\")" + " (literal \"/dev/zero\") (literal \"/dev/random\") (literal \"/dev/urandom\"))", + ] + if readable: + # ``file-map-executable`` alongside the read: being allowed to *read* a + # dynamic library is not being allowed to map its pages executable, and + # on a ``(deny default)`` profile the second denial is what actually + # stops a process. It stops it as a ``SIGABRT`` from inside dyld before + # the runtime owns stderr, so the failure arrives as a signalled child + # with no output at all rather than as anything naming a path -- which + # is how this profile read as "the launcher cannot start a child" on + # every macOS host while being a one-rule omission. + subpaths = " ".join(f"(subpath {_seatbelt_literal(path)})" for path in readable) + rules.append(f"(allow file-read* file-map-executable process-exec* {subpaths})") + if writable: + subpaths = " ".join(f"(subpath {_seatbelt_literal(path)})" for path in writable) + rules.append(f"(allow file-read* file-write* {subpaths})") + return (launcher, "-p", "\n".join(rules)) + + +def _bubblewrap_prefix(launcher: str, *, writable: Sequence[str], readable: Sequence[str]) -> tuple[str, ...]: + """A ``bwrap`` mount namespace containing only the exposure. + + ``--dev-bind / /`` -- what this module used to pass -- hands the child the + host's entire filesystem, read *and* write, and isolates the network alone. + The new root is empty: the runtime is bound read-only, the build's own + directories are bound writable, and ``/tmp`` is a tmpfs, so a path nobody + named does not exist for this child. + """ + argv = [ + launcher, + "--unshare-net", + "--unshare-ipc", + "--unshare-uts", + "--unshare-pid", + "--unshare-cgroup-try", + "--new-session", + "--die-with-parent", + "--proc", "/proc", + "--dev", "/dev", + "--tmpfs", "/tmp", + ] + for path in readable: + argv += ["--ro-bind-try", path, path] + # After the read-only runtime, so an exposure that lives under one of those + # paths is writable rather than shadowed by the read-only bind. + for path in writable: + argv += ["--bind", path, path] + argv.append("--") + return tuple(argv) + + +#: Each mechanism's prefix builder, and whether it decides on the resolved +#: path. A seatbelt rule matches what the kernel resolved to; a bind mount +#: names a destination in an empty root, where a link's own spelling is a path +#: the child still has to be able to walk. +_PREFIX_BUILDERS: Mapping[str, tuple[Callable[..., tuple[str, ...]], bool]] = { + "sandbox-exec": (_seatbelt_prefix, True), + "bwrap": (_bubblewrap_prefix, False), +} + + +def _prefix_for( + mechanism: Containment, + *, + writable: Sequence[Path | str], + readable: Sequence[Path | str], +) -> tuple[str, ...]: + builder, follow = _PREFIX_BUILDERS[mechanism.name] + return builder( + mechanism.launcher, + writable=_existing(writable, follow=follow), + readable=_existing([*_SYSTEM_READ_PATHS, *readable], follow=follow), + ) + + +def _accepted(listener: socket.socket) -> bool: + """Did anything actually connect? Drains one pending connection if so.""" + try: + connection, _ = listener.accept() + except OSError: + return False + connection.close() + return True + + +def _ran_the_probe(output: bytes, nonce: str) -> bool: + """Did this run's probe child actually execute under the launcher?""" + expected = hashlib.sha256(nonce.encode()).hexdigest() + return expected in output.decode("utf-8", "replace") + + +def _classify_probe(prefix: Sequence[str], *, cwd: str | None = None) -> str: + """Run the probe under ``prefix``: a real listener and a real planted secret. + + The secret is written to the host's temporary directory, which no exposure + this module builds ever includes, so an unconfined child reads it and a + confined one cannot see it at all. + + ``cwd`` is the directory the probe child starts in, and it belongs inside + the exposure being probed. A build's provider starts in the materialized + copy, which the boundary always exposes; a probe left in this process's own + working directory starts somewhere the boundary deliberately does not + expose, which is a launcher failure rather than a finding about the + mechanism. + """ + nonce = secrets.token_hex(16) + handle, secret = tempfile.mkstemp(prefix="code-mower-containment-probe-") + try: + os.write(handle, secrets.token_hex(32).encode()) + finally: + os.close(handle) + try: + with socket.socket() as listener: + try: + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + except OSError: # pragma: no cover - a host that cannot listen on loopback + return _UNUSABLE + # The child connects and exits; the connection waits in the backlog + # until it is accepted below, so the accept order does not matter. + listener.settimeout(1) + port = listener.getsockname()[1] + try: + completed = subprocess.run( + [*prefix, sys.executable, "-c", _CONTAINMENT_PROBE, str(port), secret, nonce], + check=False, + capture_output=True, + timeout=60, + cwd=cwd, + env={"PATH": os.environ.get("PATH", ""), **_NETWORK_DENY}, + ) + except (OSError, subprocess.SubprocessError): + return _UNUSABLE + arrived = _accepted(listener) + finally: + with contextlib.suppress(OSError): + os.unlink(secret) + # Before anything is read from the exit code: a run that cannot show its + # child executed classifies as nothing at all. This is the control that + # keeps "denied" from being the default answer for a launcher that never + # started the probe. + if not _ran_the_probe(completed.stdout, nonce): + return _UNUSABLE + observed = completed.returncode - _PROBE_BASE + if observed not in (0, 1, 2, 3): + return _UNUSABLE + read_secret = bool(observed & _PROBE_READ_SECRET) + if arrived != bool(observed & _PROBE_REACHED_LISTENER): + # The child and the listener disagree about whether a connection + # happened; treat that as a probe that proved nothing. + return _UNUSABLE + if arrived and read_secret: + return _REACHED + if not arrived and not read_secret: + return _CONTAINED + # Exactly one boundary held. A mechanism that denies sockets while leaving + # the host filesystem readable is not the boundary this module claims, and + # accepting it is the defect this classification exists to refuse. + return _UNUSABLE + + +def _prefix_confines(prefix: Sequence[str], *, cwd: str | None = None) -> bool: + """Watch a child under ``prefix`` fail to reach either thing that is really there.""" + return _classify_probe(prefix, cwd=cwd) == _CONTAINED + + +def _interpreter_read_paths() -> tuple[str, ...]: + """The minimum a probe child needs to be a running Python at all. + + Both spellings of the interpreter: a virtual environment's ``bin/python`` + is a link, and the child is launched by the name this process knows it by, + not by the name it resolves to. + """ + return tuple( + path + for path in (sys.executable, os.path.realpath(sys.executable), sys.prefix, sys.base_prefix) + if path + ) + + +def _probe_containment() -> Containment | None: + if not sys.executable: # pragma: no cover - a frozen interpreter cannot probe + return None + readable = _interpreter_read_paths() + # The scratch directory is the writable exposure *and* the directory every + # probe child starts in, control included, so the control and the candidates + # differ in the boundary and in nothing else. + with tempfile.TemporaryDirectory(prefix="code-mower-containment-") as scratch: + # The control, first: a child with no prefix must reach the listener + # *and* read the planted secret. If it cannot -- no probe interpreter, + # loopback blocked, an unreadable temporary directory -- then "could + # not" proves nothing about any candidate, and every candidate would + # pass for a boundary. Refuse the whole probe instead. + if _classify_probe((), cwd=scratch) != _REACHED: + return None + for name, path in _SANDBOX_CANDIDATES: + launcher = _trusted_launcher(path) + if launcher is None: + continue + mechanism = Containment(name=name, launcher=launcher) + prefix = _prefix_for(mechanism, writable=(scratch,), readable=readable) + if _prefix_confines(prefix, cwd=scratch): + return mechanism + return None + + +def containment_mechanism() -> Containment | None: + """The isolation mechanism this host was observed providing, if any. + + Probed once per process and cached, because the answer is a property of the + host rather than of a build. ``None`` means this host offers no mechanism + this build could *observe* denying a child both the network and the host + filesystem, and a build refuses rather than running a provider it cannot + contain. + """ + global _containment, _containment_probed + if not _containment_probed: + _containment = _probe_containment() + _containment_probed = True + return _containment + + +def containment_prefix( + *, + writable: Sequence[Path | str], + readable: Sequence[Path | str], + repository: Path, +) -> tuple[str, ...]: + """The argv prefix confining a child to ``writable`` plus a read-only runtime. + + Verified against *this* exposure before it is returned, not merely built + from it. The host probe establishes that a mechanism can confine a child + when it is handed the interpreter paths; it says nothing about the prefix a + particular build ends up with, whose readable set is the pinned provider's + install and whose writable set is that build's own directories. A widened + exposure that happened to reopen the boundary would otherwise be caught by + nothing between here and the provider. + + ``repository`` is the checkout being indexed, and it is here because the + refusals have to be applied to the *whole* readable set rather than to each + root a caller asks for. Every exposure a build requests goes through + :func:`_refuse_broad_exposure`; the read-only runtime this module adds on + top of it never did, so the set the child is really confined to was never + checked as a set. That is what let a runtime path that happened to contain + the checkout leave the live working tree readable beside the materialized + copy that exists to replace it. + + The probe's own exposure is this one plus the interpreter, because the + probe child is a Python that has to be able to start at all. That makes the + probed prefix strictly more permissive than the returned one, so a probe + that still observes containment is a sound statement about the prefix a + build actually runs under. + """ + mechanism = containment_mechanism() + if mechanism is None: + raise ContextError( + "local graph builds need an OS sandbox that denies the provider the network and " + "the host filesystem; this host offers none that could be verified" + ) + # Both spellings of everything the child could read, which is a superset of + # what either mechanism is handed: whichever spelling a mechanism decides + # on, it is checked here. + _refuse_broad_readable( + _existing([*_SYSTEM_READ_PATHS, *readable], follow=False), repository=repository + ) + prefix = _prefix_for(mechanism, writable=writable, readable=readable) + probe = _prefix_for( + mechanism, + writable=writable, + readable=(*readable, *_interpreter_read_paths()), + ) + # Inside the exposure, because that is where the confined child starts: a + # probe launched in a directory the boundary deliberately does not expose + # fails to start and says nothing about the boundary. + inside = next((str(path) for path in writable if os.path.isdir(path)), None) + if not _prefix_confines(probe, cwd=inside): + raise ContextError( + "the sandbox this build would run the local graph provider under could not be " + "observed denying it the network and the host filesystem; no generation was published" + ) + return prefix + + +def _object_name(value: Any) -> str: + """A full SHA-1 or SHA-256 object name. Abbreviations do not bind.""" + if not isinstance(value, str) or not _OBJECT_NAME.fullmatch(value): + raise ContextError("local graph revision must be a full object name") + return value + + +def _digest(value: Any) -> str: + if not isinstance(value, str) or not _DIGEST.fullmatch(value): + raise ContextError("local graph digest must be a SHA-256 hex digest") + return value + + +def _size(value: Any, maximum: int) -> int: + if type(value) is not int or not 0 <= value <= maximum: + raise ContextError("local graph size must be a bounded non-negative integer") + return value + + +#: The extraction restrictions the adopt decision was conditioned on, recorded +#: in ``docs/graphify-evaluation.md``: code-only extraction, and no clustering. +#: They are not the operator's to omit. Model-based extraction and clustering +#: are separate explicit decisions nobody has taken, and a pin that simply left +#: its ``options`` empty would have launched the provider into both. +_REQUIRED_EXTRACT_OPTIONS = ("--code-only", "--no-cluster") + +#: Options that would undo one of the above. Named exactly rather than guessed +#: at: these are the negations of the two flags this module requires, so a pin +#: carrying one is asking for behaviour the adoption conditions exclude and is +#: refused rather than silently overridden by argument order. +_CONFLICTING_EXTRACT_OPTIONS = frozenset({"--cluster", "--no-code-only"}) + +#: How many options an extraction may carry, counted on the normalized tuple -- +#: the one the launcher passes and the manifest records -- rather than on what a +#: pin file happened to spell. Counting the raw list instead would let a pin +#: pass validation and then normalize into a value its own ``as_metadata`` could +#: no longer be read back through: a build could publish a generation whose +#: manifest is rejected the moment anything reloads it, pruning the last usable +#: generation in favour of one nothing can read. +MAX_EXTRACT_OPTIONS = 16 + + +def _extraction_options(options: Iterable[str]) -> tuple[str, ...]: + """The options every extraction runs with: the required ones, then the pin's. + + Required unconditionally rather than merely validated, so a pin written + before these conditions existed -- or one with no ``options`` at all -- + still launches a restricted run. They are prepended into the pin itself + rather than added at the call site, so the manifest records what actually + ran instead of what was asked for. + + The result is a fixed point: the required flags are dropped from the input + wherever they appear and re-prepended exactly once, so normalizing an + already-normalized tuple returns it unchanged and a pin round-trips through + ``as_metadata`` and :func:`load_pin` without changing length or meaning. + """ + extra: list[str] = [] + for option in options: + name = option.split("=", 1)[0].strip() + if name in _CONFLICTING_EXTRACT_OPTIONS: + raise ContextError( + "local graph provider options may not re-enable clustering or non-code extraction" + ) + if name in _REQUIRED_EXTRACT_OPTIONS: + if option != name: + # ``--code-only=false`` is the same request as ``--no-code-only``. + raise ContextError( + "local graph provider options may not give a value to a required extraction flag" + ) + continue + extra.append(option) + normalized = (*_REQUIRED_EXTRACT_OPTIONS, *extra) + if len(normalized) > MAX_EXTRACT_OPTIONS: + raise ContextError("local graph provider options must be a bounded list") + return normalized + + +@dataclass(frozen=True) +class GraphifyPin: + """An exact provider pin. A range would let a build drift silently. + + ``wheel_sha256`` is the artifact digest recorded by the adopt decision in + ``docs/graphify-evaluation.md``. It is carried into every build manifest so + a graph built by a substituted distribution is identifiable after the fact, + which is the whole point of pinning a lookalike-prone package name. + + ``options`` always carries the required extraction restrictions, whichever + way the pin was constructed, so there is no shape of this object that could + launch an unrestricted run. + """ + + distribution: str + version: str + wheel_sha256: str + options: tuple[str, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "options", _extraction_options(self.options)) + + @property + def requirement(self) -> str: + return f"{self.distribution}=={self.version}" + + def as_metadata(self) -> dict[str, Any]: + return { + "distribution": self.distribution, + "version": self.version, + "wheel_sha256": self.wheel_sha256, + "options": list(self.options), + } + + +def load_pin(source: Mapping[str, Any]) -> GraphifyPin: + """Parse a provider pin, rejecting anything that is not one exact release.""" + if not isinstance(source, Mapping): + raise ContextError("local graph provider pin must be an object") + unknown = set(source) - {"distribution", "version", "wheel_sha256", "options"} + if unknown: + raise ContextError("local graph provider pin carries unsupported fields") + version = source.get("version") + if not isinstance(version, str) or not _VERSION.fullmatch(version): + raise ContextError("local graph provider pin must name one exact released version") + options = source.get("options", []) + # A cheap guard on the untrusted list before any of it is copied; the + # binding bound is applied to the normalized tuple in ``_extraction_options`` + # below, which is what the launcher and the manifest actually carry. + if not isinstance(options, list) or len(options) > MAX_EXTRACT_OPTIONS: + raise ContextError("local graph provider options must be a bounded list") + return GraphifyPin( + distribution=_identifier(source.get("distribution")), + version=version, + wheel_sha256=_digest(source.get("wheel_sha256")), + options=tuple(_text(option, maximum=128) for option in options), + ) + + +@dataclass(frozen=True) +class TrackedEntry: + """One tracked regular file at the bound revision.""" + + mode: str + blob: str + path: str + size: int + + +@dataclass(frozen=True) +class TrackedCensus: + """What the indexer was allowed to see, and proof of exactly which bytes. + + ``digest`` covers mode, blob name, size, and path for every entry in sorted + order. Two builds of the same commit produce the same census digest, and a + census that silently gained or lost a file produces a different one, so a + manifest's census claim is checkable without re-reading the repository. + """ + + entries: tuple[TrackedEntry, ...] + skipped: tuple[tuple[str, str], ...] + digest: str + + @property + def file_count(self) -> int: + return len(self.entries) + + @property + def total_bytes(self) -> int: + return sum(entry.size for entry in self.entries) + + +def _census_digest(entries: Iterable[TrackedEntry]) -> str: + census = hashlib.sha256() + for entry in entries: + census.update(f"{entry.mode} {entry.blob} {entry.size} {entry.path}\n".encode()) + return census.hexdigest() + + +def git_environment() -> dict[str, str]: + """The environment every Git child of a build runs in. + + One definition for both invocation paths -- the census reader and the blob + materializer -- because a boundary that only half the children observe is + not a boundary. Beyond the scrubbing an indexer gets, this denies Git the + two ways a *read* can reach the network: ``GIT_NO_LAZY_FETCH`` stops a + partial clone fetching a missing object mid-read, and an empty + ``GIT_ALLOW_PROTOCOL`` leaves no transport on the allowlist, so a fetch + that was somehow attempted anyway has nothing to attempt it over. + + ``GIT_NO_REPLACE_OBJECTS`` is the third: a ``refs/replace`` entry in the + checkout substitutes one object's bytes for another's on every read, so a + census and a materialization would bind content that the commit and tree + this manifest records do not contain. Deleting the replacement afterwards + would leave ``graph_status`` reporting ``current`` for a graph of bytes + that revision never had, because it compares object names and nothing else. + """ + environment = dict(_NETWORK_DENY) + for name in _ENVIRONMENT_ALLOWLIST: + if name in os.environ: + environment[name] = os.environ[name] + environment.update( + GIT_CONFIG_NOSYSTEM="1", + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_SYSTEM=os.devnull, + GIT_ATTR_NOSYSTEM="1", + GIT_OPTIONAL_LOCKS="0", + GIT_NO_LAZY_FETCH="1", + GIT_NO_REPLACE_OBJECTS="1", + # An empty allowlist, not an absent one: git treats the variable as the + # complete set of permitted transports, so "" permits none. + GIT_ALLOW_PROTOCOL="", + GIT_PROTOCOL_FROM_USER="0", + GIT_TERMINAL_PROMPT="0", + GIT_SSH_COMMAND="/usr/bin/false", + ) + return environment + + +#: Overrides passed on the command line because that is the only level that +#: outranks the repository's own ``.git/config``. System and global +#: configuration are dropped by the environment above, but local configuration +#: belongs to the untrusted checkout and is always read. +_GIT_SAFETY_OPTIONS: tuple[str, ...] = ( + "-c", "protocol.allow=never", + "-c", "core.fsmonitor=false", + "-c", "fetch.recurseSubmodules=no", + "-c", "uploadpack.allowFilter=false", +) + +#: Local configuration that means "objects may be missing and fetched on +#: demand". A build refuses such a checkout outright rather than relying on +#: ``GIT_NO_LAZY_FETCH``, which older Git releases do not honour. +_PARTIAL_CLONE_KEYS = r"^(extensions\.partialclone|remote\..*\.(promisor|partialclonefilter))$" + + +def _git(repository: Path, *arguments: str, capture: bool = True, permit_failure: bool = False) -> str: + """Run git with repository configuration disarmed and no way out to a network. + + A build reads an untrusted checkout. Local, global, and system + configuration can install clean/smudge filters, alternate object stores, + and hook paths, any of which would run code or reach outside the + repository during what looks like a read. This drops all three, and + ``git_environment`` closes the transports a read could otherwise use. + """ + try: + completed = subprocess.run( + ["git", "-C", str(repository), "--no-optional-locks", *_GIT_SAFETY_OPTIONS, *arguments], + check=not permit_failure, + capture_output=capture, + text=True, + env=git_environment(), + ) + except (OSError, UnicodeError, subprocess.SubprocessError): + raise ContextError("local graph build could not read the target repository") from None + if permit_failure and completed.returncode != 0: + return "" + return completed.stdout + + +#: The longest NUL-delimited ``ls-tree`` record a census will hold before it has +#: seen the delimiter that ends it. A record is fixed-width metadata plus one +#: path, and Git's own path ceiling is far under this, so a longer run of bytes +#: means the stream is not the one this build asked for. +MAX_CENSUS_RECORD_BYTES = 16 * 1024 + +#: How much of the listing to read at a time. Small enough that a refusal costs +#: one chunk rather than a whole repository's metadata. +_CENSUS_CHUNK_BYTES = 64 * 1024 + + +def _stop_reader(process: subprocess.Popen[bytes]) -> None: + """Stop a streaming Git child and reap it, whatever the caller is doing. + + Closing the pipe first is what makes an early refusal cheap: Git writes into + a broken pipe and exits on its own, rather than being left to finish + enumerating a tree nobody is going to read. + """ + if process.stdout is not None: + with contextlib.suppress(OSError): + process.stdout.close() + if process.poll() is None: + with contextlib.suppress(OSError): + process.kill() + with contextlib.suppress(subprocess.TimeoutExpired): + process.wait(timeout=_REAP_TIMEOUT_SECONDS) + + +@contextlib.contextmanager +def _git_records(repository: Path, *arguments: str) -> Iterator[Iterator[str]]: + """Yield the NUL-delimited records of one Git child, one at a time. + + ``subprocess.run`` would hold the whole listing in memory before the first + budget could be checked, so a tree far past every census bound would exhaust + this process instead of being refused at the bound. Streaming lets the + consumer stop at the record that breaks its budget; leaving the child to + this context manager means it is terminated there rather than whenever a + generator happens to be collected. + """ + try: + process = subprocess.Popen( + ["git", "-C", str(repository), "--no-optional-locks", *_GIT_SAFETY_OPTIONS, *arguments], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=git_environment(), + ) + except (OSError, subprocess.SubprocessError): + raise ContextError("local graph build could not read the target repository") from None + try: + yield _read_records(process) + finally: + _stop_reader(process) + + +def _read_records(process: subprocess.Popen[bytes]) -> Iterator[str]: + stream = process.stdout + assert stream is not None + pending = b"" + while True: + chunk = stream.read(_CENSUS_CHUNK_BYTES) + if not chunk: + break + pending += chunk + while True: + record, delimiter, rest = pending.partition(b"\0") + if not delimiter: + break + if len(record) > MAX_CENSUS_RECORD_BYTES: + raise ContextError("local graph build could not read the repository census") + pending = rest + yield _record_text(record) + # The same bound on what has *not* been delimited yet: a stream with no + # delimiter in it would otherwise grow a chunk at a time forever. + if len(pending) > MAX_CENSUS_RECORD_BYTES: + raise ContextError("local graph build could not read the repository census") + if pending: + # ``-z`` terminates every record, so a trailing remainder is a stream + # that stopped mid-record: a killed child, or output this build did not + # ask for. Either way the census it would produce is incomplete. + raise ContextError("local graph build could not read the repository census") + if process.wait() != 0: + raise ContextError("local graph build could not read the target repository") + + +def _record_text(record: bytes) -> str: + try: + return record.decode("utf-8") + except UnicodeDecodeError: + raise ContextError("local graph build could not read the repository census") from None + + +def refuse_lazy_object_fetch(repository: Path) -> None: + """Refuse a partial clone, where reading the tree can call out to a remote. + + ``ls-tree`` and ``cat-file`` look like pure local reads, and in a full + clone they are. In a partial clone a missing object is fetched from the + promisor remote on demand -- during the build, over a transport the + repository configured, outside the sandbox the provider runs in. There is + no bounded way to prove ahead of time which objects are present, so the + build declines the whole repository shape instead. + """ + declared = _git( + repository, + "config", + "--local", + "--name-only", + "--get-regexp", + _PARTIAL_CLONE_KEYS, + permit_failure=True, + ) + if declared.strip(): + raise ContextError( + "local graph builds refuse a partial clone: reading its tree can fetch objects " + "from a remote during the build; use a full clone of this checkout" + ) + + +def resolve_revision(repository: Path, revision: str = "HEAD") -> tuple[str, str]: + """Return the full ``(commit, tree)`` names a build would bind. + + The tree is resolved separately rather than derived, because it is what a + consumer actually compares: two commits with different messages or parents + over identical content share a tree, and a graph of that content is still + accurate for both. + """ + name = _text(revision) + if name.startswith("-"): + # Otherwise the revision reaches ``git rev-parse`` as an option. + raise ContextError("local graph revision must not begin with an option marker") + commit = _object_name(_git(repository, "rev-parse", "--verify", f"{name}^{{commit}}").strip()) + tree = _object_name(_git(repository, "rev-parse", "--verify", f"{commit}^{{tree}}").strip()) + return commit, tree + + +def _private_state_reason(path: str) -> str | None: + """Why this tracked path may not enter the graph, or ``None`` if it may. + + Provider state keeps its own reason because the consequence is specific -- + the provider resuming from a cache of content this build never saw -- while + ``.git`` and ``.code-mower`` are private state of a different kind: version + control's own storage, and this tool's packets, evidence and lane records. + Every segment is tested, case-folded, so ``vendor/.git`` and a nested + ``docs/.CODE-MOWER`` are as excluded as the top-level ones. + """ + segments = [segment.casefold() for segment in path.split("/")] + if any(segment in _PROVIDER_STATE_ROOTS for segment in segments): + return "provider state" + if any(segment in _PRIVATE_STATE_ROOTS for segment in segments): + return "private state" + return None + + +def read_tracked_census(repository: Path, commit: str) -> TrackedCensus: + """List the tracked regular files of one commit, with their blob sizes. + + Reads the commit's tree, never the working tree or the index, so an + uncommitted edit, an untracked scratch file, and an ignored secret are all + invisible here by construction rather than by filtering. + + Committed private state -- the provider's own index, ``.git``, and this + tool's ``.code-mower`` -- is recorded as skipped rather than carried: it is + excluded from the census, so it is excluded from the census digest too, and + a build over a repository that tracks a ``.graphify`` directory binds a + census that says so instead of quietly indexing somebody else's graph. + + The listing is consumed as it arrives rather than captured whole: every + budget here is checked against the records seen so far, so a tree past one + of them is refused at that record, with the reader stopped, instead of + after a repository's worth of metadata has been buffered. + """ + refuse_lazy_object_fetch(repository) + entries: list[TrackedEntry] = [] + skipped: list[tuple[str, str]] = [] + total = 0 + + def skip(path: str, reason: str) -> None: + if len(skipped) >= MAX_SKIPPED_PATHS: + raise ContextError("skipped path census exceeds the local graph budget") + skipped.append((path, reason)) + + with _git_records( + repository, + "ls-tree", + "-r", + "-z", + "--long", + "--full-tree", + _object_name(commit), + ) as records: + for record in records: + if not record: + continue + metadata, _, path = record.partition("\t") + fields = metadata.split() + if len(fields) != 4 or not path: + raise ContextError("local graph build could not read the repository census") + mode, kind, blob, raw_size = fields + if mode in _SKIPPED_MODES: + skip(path, _SKIPPED_MODES[mode]) + continue + excluded = _private_state_reason(path) + if excluded is not None: + skip(path, excluded) + continue + if mode not in _REGULAR_MODES or kind != "blob": + skip(path, "unsupported") + continue + if len(entries) >= MAX_TRACKED_FILES: + raise ContextError("tracked file census exceeds the local graph budget") + size = int(raw_size) if raw_size.isdigit() else -1 + if not 0 <= size <= MAX_BLOB_BYTES: + raise ContextError("tracked file exceeds the local graph per-file budget") + total += size + if total > MAX_TRACKED_BYTES: + raise ContextError("tracked content exceeds the local graph budget") + entries.append(TrackedEntry(mode=mode, blob=_object_name(blob), path=path, size=size)) + entries.sort(key=lambda entry: entry.path) + return TrackedCensus( + entries=tuple(entries), + skipped=tuple(sorted(skipped)), + digest=_census_digest(entries), + ) + + +def _safe_relative(path: str) -> Path: + """Reject any census path that would escape the materialization root. + + Git does not normally produce these, but a build must not depend on that: + the destination is created by this process and everything written into it + is checked here first. + """ + if ( + not path + or path.startswith("/") + or "\\" in path + or any(segment in {"", ".", ".."} for segment in path.split("/")) + # Private state is skipped by the census, so a census that still + # carries it was not built by ``read_tracked_census``. Refuse rather + # than write ``.git`` or ``.code-mower`` into the tree the provider is + # about to read, or seed the directory it is about to write into. + # Every segment, not just the first: a vendored submodule's + # ``vendor/.git`` is as private as the top-level one. Case-folded + # because APFS and NTFS name the same directory ``.GIT``. + or _private_state_reason(path) is not None + ): + raise ContextError("tracked path must stay inside the materialized checkout") + return Path(*path.split("/")) + + +def materialize_tracked_files(repository: Path, census: TrackedCensus, destination: Path) -> int: + """Write the census's blobs into ``destination``. Returns bytes written. + + ``destination`` must not already exist: an immutable materialization is one + this build created and fully owns, so there is no prior content to + reconcile and no possibility of reusing a directory somebody else can + write. Blob content comes from ``git cat-file --batch`` in one child + process rather than one per file. + """ + refuse_lazy_object_fetch(repository) + if destination.exists(): + raise ContextError("local graph materialization requires a fresh private directory") + destination.mkdir(mode=0o700, parents=True) + if not census.entries: + return 0 + written = 0 + process = subprocess.Popen( + ["git", "-C", str(repository), "--no-optional-locks", *_GIT_SAFETY_OPTIONS, "cat-file", "--batch"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=git_environment(), + ) + try: + assert process.stdin is not None and process.stdout is not None + for entry in census.entries: + target = destination / _safe_relative(entry.path) + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + process.stdin.write(entry.blob.encode() + b"\n") + process.stdin.flush() + header = process.stdout.readline().decode("utf-8", "replace").split() + if len(header) != 3 or header[1] != "blob" or not header[2].isdigit(): + raise ContextError("local graph materialization could not read tracked content") + size = int(header[2]) + if size != entry.size: + raise ContextError("tracked content changed during materialization") + payload = process.stdout.read(size) + if len(payload) != size or process.stdout.read(1) != b"\n": + raise ContextError("local graph materialization was truncated") + # 0o600 regardless of the tracked mode: the indexer reads this copy + # and never executes it, and an executable bit here would only + # widen what a provider process can do with the staging directory. + handle = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + with os.fdopen(handle, "wb") as stream: + stream.write(payload) + written += size + except BaseException: + process.kill() + raise + finally: + for stream in (process.stdin, process.stdout): + if stream is not None and not stream.closed: + stream.close() + try: + process.wait(timeout=60) + except subprocess.TimeoutExpired: # pragma: no cover - unresponsive child + process.kill() + return written + + +def scrubbed_environment(*, home: Path, temporary: Path) -> dict[str, str]: + """Build the provider process environment from an allowlist. + + An indexer inherits nothing from the operator's session but the variables + it needs to find executables and write scratch files. ``HOME`` is + redirected into the build's own private directory so a provider's + configuration, cache, or credential lookup lands there instead of reading + or writing the operator's real home. + """ + environment = { + name: os.environ[name] for name in _ENVIRONMENT_ALLOWLIST if name in os.environ + } + environment.update(_NETWORK_DENY) + environment.update( + HOME=str(home), + TMPDIR=str(temporary), + XDG_CONFIG_HOME=str(home / "config"), + XDG_CACHE_HOME=str(home / "cache"), + XDG_DATA_HOME=str(home / "data"), + LC_ALL="C", + LANG="C", + TZ="UTC", + ) + return environment + + +@dataclass(frozen=True) +class IndexRequest: + """What an indexer is given: a frozen copy, a scratch area, and a pin.""" + + source_root: Path + output_path: Path + environment: Mapping[str, str] + pin: GraphifyPin + commit: str + tree: str + #: The scratch directories the provider may write to besides the copy -- + #: the redirected ``HOME`` and ``TMPDIR``. Named here rather than inferred + #: because they are also exactly what the filesystem boundary exposes: a + #: directory the environment points at but the sandbox does not expose is a + #: provider that cannot start. + writable: tuple[Path, ...] = () + + +@dataclass(frozen=True) +class IndexResult: + """What an indexer reports back. ``completeness`` is its own admission.""" + + completeness: str = COMPLETE + indexed_files: int = 0 + notes: tuple[str, ...] = () + + +def _resolved_executable(executable: str) -> str: + """Bind the provider to one absolute path, decided in the invocation directory. + + The provider runs with its working directory set to the materialized copy, + so ``--indexer .venv/bin/graphify`` would otherwise be looked up inside the + frozen source tree, where the operator's install is not. + + A bare command name is resolved here too, rather than left for the launch to + look up again. ``PATH`` is not independent of the child's directory: an entry + on it may itself be relative -- ``PATH=provider-venv/bin:$PATH`` is an + ordinary thing to have in a shell sitting in a project -- and a relative + entry names a different directory once the child starts in the materialized + copy. Containment then drew its boundary around the install this process + found while the launch searched again from somewhere else, so a correctly + installed provider failed to launch at all; worse, a repository that happens + to carry that path would answer the second search with a *tracked file*, + which is a build executing content it was only ever meant to read. One + lookup, in the directory the operator invoked from, and both the exposure + and the argv are that one absolute path. + """ + if not isinstance(executable, str) or not executable: + raise ContextError("local graph provider executable must be named") + separators = [os.sep, os.altsep] if os.altsep else [os.sep] + if any(separator in executable for separator in separators): + return str(Path(executable).resolve()) + located = shutil.which(executable) + if not located: + raise ContextError( + "local graph provider executable could not be found on PATH; name it by path " + "or install the pinned provider where this process can see it" + ) + return str(Path(located).resolve()) + + +#: The directory a console script sits in, by platform convention. Named so an +#: install root is recognised rather than guessed at from depth alone. +_VENV_SCRIPT_DIRECTORIES = ("bin", "Scripts") + +#: What a runtime needs readable *inside* an install prefix, for the case where +#: the prefix itself is too wide to expose. ``/usr`` is the ordinary base for an +#: environment created from the system Python, and exposing it whole is the +#: exposure this module just spent a list narrowing away; these are the +#: directories an interpreter and its standard library actually live in, and on +#: a prefix like ``/usr`` they are already the read-only runtime every child +#: gets, so the narrowing adds nothing rather than widening anything. +_RUNTIME_SUBDIRECTORIES = ("lib", "lib64", "lib32", "libexec", "bin", "share") + +#: ``pyvenv.cfg`` is a handful of ``key = value`` lines. Bounded at the stream +#: anyway: it is a file inside somebody else's install, and a build should not +#: be able to be stopped by reading one. +_MAX_VENV_CONFIG_BYTES = 65_536 + +#: Where a ``pyvenv.cfg`` records the interpreter its environment was created +#: from, in the spellings the three tools that write the file actually use, and +#: what each spelling names. ``virtualenv`` writes ``base-prefix`` and +#: ``base-exec-prefix``; ``uv`` writes those as well; the standard library's +#: ``venv`` writes ``home`` always and ``executable`` since 3.11. A prefix is +#: used as it stands, an executable is the interpreter binary and its prefix is +#: the directory above its script directory, and ``home`` is the script +#: directory itself. +_VENV_BASE_PREFIX_KEYS = ("base-prefix", "base-exec-prefix") +_VENV_BASE_EXECUTABLE_KEYS = ("base-executable", "executable") +_VENV_BASE_SCRIPT_DIRECTORY_KEY = "home" + + +def _under(path: Path, root: Path) -> bool: + return path == root or path.is_relative_to(root) + + +def _read_venv_config(root: Path) -> dict[str, str]: + """The ``key = value`` pairs of an environment's ``pyvenv.cfg``. + + Unparseable lines are skipped rather than fatal. The file is a record left + by whichever tool created the environment, and a key this module does not + know about is not a reason to refuse an install that works. + """ + path = root / "pyvenv.cfg" + try: + with path.open("rb") as stream: + raw = stream.read(_MAX_VENV_CONFIG_BYTES + 1) + except OSError: + return {} + if len(raw) > _MAX_VENV_CONFIG_BYTES: + raise ContextError( + "local graph provider's pyvenv.cfg is implausibly large; refusing to derive " + "a containment boundary from it" + ) + config: dict[str, str] = {} + for line in raw.decode("utf-8", "replace").splitlines(): + key, separator, value = line.partition("=") + if not separator: + continue + config.setdefault(key.strip().lower(), value.strip()) + return config + + +def _prefix_of_interpreter(executable: Path) -> Path: + """The install prefix holding an interpreter binary. + + ``/bin/python3.13`` on POSIX, ``\\python.exe`` on Windows. + The script-directory test is what distinguishes them, so a layout that is + neither is left at the directory the binary sits in rather than climbing a + level it cannot justify. + """ + if executable.parent.name in _VENV_SCRIPT_DIRECTORIES: + return executable.parent.parent + return executable.parent + + +def _provider_base_prefixes(root: Path, *, repository: Path) -> tuple[str, ...]: + """The base runtime *the provider's own environment* was created from. + + This used to be ``sys.base_prefix``, which is the interpreter running Code + Mower. The two coincide only when the provider was pinned with the same + Python this process happens to be running under. A provider pinned with a + ``uv``-managed or otherwise separately installed interpreter -- an entirely + ordinary way to pin one -- got somebody else's runtime exposed and its own + left out of the child's filesystem view, so every build failed on a + correctly pinned install, and failed from inside the loader rather than + with anything naming a path. + + So the environment is asked instead of assumed: ``pyvenv.cfg`` records the + interpreter that created it, and that record is what gets exposed. Each + candidate is held to exactly the refusals the environment root is held to + -- a base prefix that is the operator's home or the checkout is no narrower + for having been read out of a file rather than guessed -- and one already + inside the read-only system runtime adds nothing. + """ + config = _read_venv_config(root) + candidates: list[Path] = [] + for key in _VENV_BASE_PREFIX_KEYS: + value = config.get(key) + if value and os.path.isabs(value): + candidates.append(Path(value)) + for key in _VENV_BASE_EXECUTABLE_KEYS: + value = config.get(key) + if value and os.path.isabs(value): + candidates.append(_prefix_of_interpreter(Path(value))) + value = config.get(_VENV_BASE_SCRIPT_DIRECTORY_KEY) + if value and os.path.isabs(value): + script_directory = Path(value) + candidates.append( + script_directory.parent + if script_directory.name in _VENV_SCRIPT_DIRECTORIES + else script_directory + ) + system = tuple(Path(os.path.realpath(path)) for path in _SYSTEM_READ_PATHS) + exposed: list[str] = [] + seen: set[Path] = set() + resolved_any = False + for candidate in candidates: + base = Path(os.path.realpath(candidate)) + if base in seen: + continue + seen.add(base) + if not base.is_dir(): + # A stale record -- a base interpreter moved or removed since the + # environment was created. Not fatal on its own: another key may + # still name a live one, and only all of them failing is a broken + # pin. + continue + resolved_any = True + # Already covered: inside the read-only runtime every child gets, or + # inside the environment root that is being exposed anyway. + if any(_under(base, path) for path in system) or _under(base, Path(os.path.realpath(root))): + continue + _refuse_broad_exposure(base, repository=repository) + if any(_under(path, base) for path in system): + # A prefix that *contains* the read-only runtime is not a narrow + # install: ``/usr`` is what an environment created from the system + # Python records, and exposing it whole hands the provider + # ``/usr/local``, ``/usr/src``, and anything else the host keeps + # there. The runtime directories inside it are exposed instead -- + # which, for a prefix like that one, are the system paths every + # child already has, so nothing is added at all. + for name in _RUNTIME_SUBDIRECTORIES: + runtime = base / name + if runtime.is_dir() and not any(_under(runtime, path) for path in system): + exposed.append(str(runtime)) + continue + exposed.append(str(base)) + if not resolved_any: + raise ContextError( + "local graph provider's virtual environment does not record a usable base " + "interpreter in its pyvenv.cfg, so the runtime it needs cannot be exposed to " + "the sandbox; recreate the environment with the interpreter the provider is " + "pinned for" + ) + return tuple(exposed) + + +def _is_broad_exposure(root: Path, *, repository: Path) -> bool: + """Would exposing ``root`` hand over somebody's whole world? + + The filesystem root, the operator's home, the checkout being indexed, and + every ancestor of either: each of these is a directory whose contents are + exactly what this boundary exists to keep away from the provider. A root + that *is* one of them is not a narrow install however it was arrived at, + and a root *inside the checkout* is the live working tree -- ignored + secrets and all -- which the materialized copy exists precisely to avoid + showing anyone. + """ + resolved = Path(os.path.realpath(root)) + checkout = Path(os.path.realpath(repository)) + try: + home = Path(os.path.realpath(Path.home())) + except (OSError, RuntimeError): # pragma: no cover - a host with no home + home = None + refused = {Path(resolved.anchor), checkout, *checkout.parents} + if home is not None: + refused.update({home, *home.parents}) + return resolved in refused or _under(resolved, checkout) + + +def _refuse_broad_exposure(root: Path, *, repository: Path) -> None: + """Refuse an exposure root a provider install cannot justify.""" + if _is_broad_exposure(root, repository=repository): + raise ContextError( + "local graph provider must be pinned into its own virtual environment; " + "exposing its install would expose the filesystem root, your home directory, " + "or the checkout being indexed" + ) + + +def _refuse_broad_readable(readable: Iterable[str], *, repository: Path) -> None: + """Hold the *whole* readable set to the refusals one exposure root is held to. + + Each root a build asks for is checked as it is derived, and that was the + only check there was: the read-only runtime is added afterwards and + unconditionally, so the set the child ends up confined to was never examined + as a set. A runtime path that contained the checkout -- a checkout under + ``/usr/local/src`` when this module exposed ``/usr``, say -- therefore left + the live working tree readable beside the materialized copy that exists + precisely so the provider never sees it, and no amount of care in deriving + the *provider's* exposure could notice. + + So the final set is checked, whatever put a path in it. A build whose + runtime exposure would reach the operator's home or the checkout is refused + rather than narrowed silently: on an ordinary host nothing here is close to + either, and on a host where one of them is, the boundary this module claims + does not hold and saying so is the only honest answer. + """ + for path in readable: + if _is_broad_exposure(Path(path), repository=repository): + raise ContextError( + "the read-only runtime this build would expose to the local graph provider " + "reaches the filesystem root, your home directory, or the checkout being " + "indexed, so the provider could read the working tree the materialized copy " + "replaces; no generation was published" + ) + + +def _provider_read_paths(command: str, *, repository: Path) -> tuple[str, ...]: + """The install the pinned provider needs to be readable, and nothing beside it. + + The executable's parent and grandparent used to be exposed on the reasoning + that a console script lives in a virtual environment's ``bin``. That is a + guess about a layout, not knowledge of one, and the guess is wrong in the + directions that matter most: ``~/bin/graphify`` makes the grandparent the + operator's home, ``/opt/graphify`` makes it ``/``, and a provider inside the + checkout makes it the live working tree the materialized copy exists to + avoid showing anybody. The boundary was then drawn around whatever that + came out to, and the probe -- which exercises only the interpreter paths -- + never touched it. + + So the layout is *proved* instead. An install root is a directory holding a + ``pyvenv.cfg`` whose script directory holds this executable: a virtual + environment, which is what pinning a provider produces, and whose root is + narrow by construction. A provider that already lives inside the read-only + system runtime needs no extra exposure at all and gets none. Anything else + is refused with an instruction rather than exposed as a guess, and whatever + root is arrived at is put through ``_refuse_broad_exposure`` regardless -- + a proof of layout is not a proof that the layout is narrow. + """ + located = command if os.path.isabs(command) else shutil.which(command) + if not located: + raise ContextError("local graph provider executable could not be located for containment") + real = Path(os.path.realpath(located)) + # Both spellings of the executable itself: the child is launched by the name + # this process resolved the command to, and a console script reaches its + # environment through that environment's own spelling rather than through + # whatever the link points at. + spellings = tuple(dict.fromkeys((str(located), str(real)))) + system = tuple(Path(os.path.realpath(path)) for path in _SYSTEM_READ_PATHS) + if any(_under(real, path) for path in system): + # Already inside the read-only runtime every child gets. Adding the + # enclosing prefix would widen that exposure, not narrow it. + return spellings + root = real.parent.parent + if real.parent.name not in _VENV_SCRIPT_DIRECTORIES or not (root / "pyvenv.cfg").is_file(): + raise ContextError( + "local graph provider must be pinned into its own virtual environment so its " + "install can be exposed to the sandbox without exposing anything around it" + ) + _refuse_broad_exposure(root, repository=repository) + # The base interpreter a virtual environment was created from lives outside + # it and is what its ``bin/python`` points at, so it is exposed too -- read + # out of *this* environment's own ``pyvenv.cfg`` rather than taken from the + # interpreter Code Mower happens to be running under, which is a different + # installation whenever the provider was pinned with a different Python. + return (*spellings, str(root), *_provider_base_prefixes(root, repository=repository)) + + +#: Where an installed distribution records its own identity (PEP 376). The +#: directory is named ``-.dist-info``, but the name on the +#: directory is not what is read: a directory can be renamed and ``METADATA`` +#: is what the installer wrote. +_DIST_INFO_SUFFIX = ".dist-info" + +#: Where the environment an executable belongs to keeps those records, relative +#: to the prefix the script directory sits in. Both spellings a virtual +#: environment uses and the two a system install uses are named, because +#: ``_provider_read_paths`` accepts a provider from either. +_SITE_PACKAGES_GLOBS = ( + "lib/python*/site-packages", + "lib64/python*/site-packages", + "lib/python*/dist-packages", + "Lib/site-packages", +) + +#: Enough of a ``METADATA`` file to hold its header block. The rest of that +#: file is the project's long description -- a README, sometimes a large one -- +#: and no header this reads lives past the first blank line. +_MAX_DIST_METADATA_BYTES = 65_536 + +#: A ``RECORD`` lists every file its distribution installed, so it is large for +#: a large package and bounded for the same reason every other foreign file +#: here is: it is read out of somebody else's install. +_MAX_DIST_RECORD_BYTES = 8 * 1024 * 1024 + +#: How much of the provider executable is hashed against its recorded digest. +#: A console script is a few hundred bytes and a compiled launcher a few +#: megabytes; anything past this is not an install this check can speak to. +_MAX_PROVIDER_EXECUTABLE_BYTES = 64 * 1024 * 1024 + +#: Runs of the separators PEP 503 collapses, for comparing a pinned +#: distribution name against an installed one. ``Graphify_Y`` and ``graphify-y`` +#: are one distribution, and a pin must not fail against its own install over +#: the spelling an installer happened to write. +_NAME_SEPARATORS = re.compile(r"[-_.]+") + + +def _normalized_distribution(name: str) -> str: + return _NAME_SEPARATORS.sub("-", name.strip()).lower() + + +def _reportable(value: str) -> str: + """A short printable spelling of something read out of a foreign install. + + Installed metadata is not indexed content, but it is not this process's + text either, and it ends up in an operator-facing message. Bounded and + stripped of anything unprintable so a crafted ``METADATA`` cannot rewrite + the terminal the refusal is read in. + """ + printable = "".join(character if character.isprintable() else "?" for character in value[:64]) + return printable.strip() or "an unnamed distribution" + + +def _site_package_roots(executable: Path) -> tuple[Path, ...]: + """Where the environment this executable belongs to keeps its installs.""" + prefix = executable.parent.parent + roots: list[Path] = [] + for pattern in _SITE_PACKAGES_GLOBS: + roots.extend(candidate for candidate in sorted(prefix.glob(pattern)) if candidate.is_dir()) + return tuple(dict.fromkeys(roots)) + + +def _dist_info_headers(dist_info: Path) -> dict[str, str]: + """The ``METADATA`` header block, lowercased keys, first spelling wins.""" + try: + with (dist_info / "METADATA").open("rb") as stream: + raw = stream.read(_MAX_DIST_METADATA_BYTES) + except OSError: + return {} + headers: dict[str, str] = {} + for line in raw.decode("utf-8", "replace").splitlines(): + if not line.strip(): + # The header block ends at the first blank line. Everything after + # it is the description, which may contain anything at all, + # including lines that look like headers. + break + if line[:1] in (" ", "\t"): + continue + key, separator, value = line.partition(":") + if separator: + headers.setdefault(key.strip().lower(), value.strip()) + return headers + + +def _record_entries(dist_info: Path) -> tuple[tuple[str, str], ...]: + """``(installed path, sha256 hex or "")`` for every file a distribution wrote. + + The digest is recorded base64url-encoded without padding, and is absent for + some entries by design -- ``RECORD`` cannot record its own hash. An entry + whose digest is missing or in an algorithm this does not read comes back + with an empty one rather than being dropped: it still proves ownership, + which is the first thing this file is read for. + """ + try: + with (dist_info / "RECORD").open("rb") as stream: + raw = stream.read(_MAX_DIST_RECORD_BYTES + 1) + except OSError: + return () + if len(raw) > _MAX_DIST_RECORD_BYTES: + raise ContextError( + "the local graph provider's installed RECORD is implausibly large; refusing to " + "check the pin against it" + ) + entries: list[tuple[str, str]] = [] + try: + for row in csv.reader(io.StringIO(raw.decode("utf-8", "replace"))): + if not row or not row[0]: + continue + algorithm, _, encoded = (row[1] if len(row) > 1 else "").partition("=") + digest = "" + if algorithm == "sha256" and encoded: + try: + digest = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)).hex() + except ValueError: + digest = "" + entries.append((row[0], digest)) + except csv.Error: + # A ``RECORD`` this cannot parse claims nothing, which leaves the + # executable owned by no distribution and the build refused. Failing + # closed on an unreadable install beats accepting an unchecked one. + return () + return tuple(entries) + + +def _installed_owner(executable: Path) -> tuple[Path, str] | None: + """The distribution that installed this executable, and the digest it recorded. + + Ownership is read from ``RECORD`` rather than guessed from the file's name. + A pinned release and a lookalike can both ship a console script called + ``graphify``, and an environment is free to hold both; what the pin has to + be checked against is the distribution that wrote *this* file. + """ + target = str(executable) + for site_packages in _site_package_roots(executable): + for dist_info in sorted(site_packages.glob("*" + _DIST_INFO_SUFFIX)): + if not dist_info.is_dir(): + continue + for recorded, digest in _record_entries(dist_info): + if os.path.realpath(site_packages / recorded) == target: + return dist_info, digest + return None + + +def _executable_digest(executable: Path) -> str: + digest = hashlib.sha256() + read = 0 + try: + with executable.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + read += len(chunk) + if read > _MAX_PROVIDER_EXECUTABLE_BYTES: + raise ContextError( + "the local graph provider's executable is implausibly large; refusing to " + "check it against the digest its installer recorded" + ) + digest.update(chunk) + except OSError: + raise ContextError("the local graph provider's executable could not be read") from None + return digest.hexdigest() + + +def _verify_provider_installation(command: str, *, pin: GraphifyPin) -> None: + """Prove the executable about to run is the pinned release, before it runs. + + Every generation's manifest records the pin as the provenance of the bytes + in it, and until this check existed that record was a copy of what the + operator typed: ``--indexer`` named an install, ``--pin-file`` named a + release, and nothing compared the two. A build could publish a manifest + naming one release while a different one -- or a different distribution + that happens to answer to ``extract`` -- produced the graph, and neither + ``status`` nor the manifest could tell afterwards. For a package whose name + differs from the repository's by one character, that is the substitution + the pin exists to make identifiable. + + The install is *read*, not the provider *asked*. ``graphify --version`` + would mean launching the very executable whose identity is in question, + outside the sandbox that exists to confine it, and then believing what it + printed about itself. + + What an unpacked install can answer is identity, not provenance of the + artifact: nothing on disk retains the wheel it came from, so + ``wheel_sha256`` stays the operator's record of which artifact they + installed rather than something checkable here. The three things that are + checkable are checked -- which distribution owns this executable, at which + version, and whether the file still matches the digest its installer + recorded for it. + """ + executable = Path(os.path.realpath(command)) + owner = _installed_owner(executable) + if owner is None: + raise ContextError( + "the executable named for the local graph provider belongs to no distribution " + "installed in its environment, so it cannot be checked against the pin; install " + "the pinned release and name the console script that install provides" + ) + dist_info, recorded_digest = owner + headers = _dist_info_headers(dist_info) + installed, version = headers.get("name", ""), headers.get("version", "") + if not installed or not version: + raise ContextError( + "the local graph provider's install records no name and version of its own, so the " + "pin cannot be checked against it; no generation was published" + ) + named = _normalized_distribution(installed) == _normalized_distribution(pin.distribution) + # The version is compared exactly, as the installer recorded it: a pin is + # one release, and deciding that ``1.0`` and ``1.0.0`` are the same release + # is a version-comparison policy this has no business inventing. + if not named or version != pin.version: + raise ContextError( + f"the installed local graph provider is {_reportable(installed)} " + f"{_reportable(version)}, not the pinned {pin.distribution} {pin.version}; " + "no generation was published" + ) + if recorded_digest and _executable_digest(executable) != recorded_digest: + raise ContextError( + "the local graph provider's executable no longer matches the digest its installer " + "recorded, so the pinned install has been modified in place; no generation was " + "published" + ) + + +#: The subcommand the evaluated release exposes, recorded in +#: ``docs/graphify-evaluation.md``: the clean-room run indexed with +#: ``extract --code-only --no-cluster --max-workers 4``. There is no +#: ``--source``/``--output`` pair to hand it; ``extract`` reads the directory +#: it is run in and writes its state beside those sources, which is why the +#: child's working directory is the materialized copy and why the adapter +#: collects an artifact afterwards rather than naming one up front. +_PROVIDER_EXTRACT = "extract" + +#: The provider's own record of what it processed. Completeness is read from +#: here, never inferred from an exit status: the clean-room run recorded 54 +#: manifest entries requeued by a repeat that exited zero in 1.63 s. +_PROVIDER_REPORT_NAMES = ("manifest.json", "index.json", "report.json") + +#: Counters whose presence above zero means the provider did not finish. Any +#: one of them, not all: a report that admits requeued entries is a partial +#: build however healthy the rest of it looks. +_INCOMPLETE_COUNTERS = ("requeued", "pending", "failed", "errors", "incomplete") + +#: Where the provider reports how many files it actually indexed. +_INDEXED_COUNTERS = ("indexed_files", "code_files", "files", "entries") + +#: An affirmative claim that the run finished, in either shape a report can +#: carry one. Nothing else counts: an empty object, or one whose schema this +#: adapter does not recognize, says nothing about completion and is therefore +#: not evidence of it. +_COMPLETION_FLAGS = ("complete", "completed", "finished") +#: Narrow on purpose: a field that names the run's state, not one that might +#: carry a path or a message, so an unrecognized value here is a real +#: non-completion rather than an adapter that read the wrong field. +_COMPLETION_STATUS_FIELDS = ("status", "state") +_COMPLETION_STATUS_VALUES = frozenset( + {"complete", "completed", "success", "succeeded", "ok", "finished", "done"} +) + + +def _refuse_pre_existing_provider_state(source_root: Path) -> None: + """Refuse to extract on top of index state this build did not produce. + + The census excludes committed provider state, so in a build from this + module nothing is here. This is the second check rather than the only one: + the source root is an argument, and the whole point of resolving the state + directory afterwards is to treat what is found as freshly produced output. + A directory that predates the run would let the provider resume from a + cache of content it was never shown, and would be collected as if it were + this commit's graph. + """ + for name in _PROVIDER_STATE_DIRECTORIES: + if (source_root / name).exists() or (source_root / name).is_symlink(): + raise ContextError( + "local graph build refuses to extract over pre-existing provider state; " + "no generation was published" + ) + + +def _provider_state_directory(source_root: Path) -> Path: + """The state directory the provider wrote during this run. + + Only reachable after ``_refuse_pre_existing_provider_state``, so whichever + of the two names is present was created by the run that just finished. + """ + for name in _PROVIDER_STATE_DIRECTORIES: + candidate = source_root / name + if candidate.is_dir() and not candidate.is_symlink(): + return candidate + raise ContextError("local graph provider wrote no index state; no generation was published") + + +def _provider_report(state_directory: Path) -> Mapping[str, Any] | None: + """The provider's completion evidence, or ``None`` if it left none. + + Bounded at the stream, not after the fact: the report is provider output of + unknown size, and reading it whole to slice it afterwards would let it + exhaust this process before any budget was consulted. Anything longer than + a manifest is rejected outright rather than parsed from a prefix, which + would be a different document than the one the provider wrote. + """ + for name in _PROVIDER_REPORT_NAMES: + path = state_directory / name + if not path.is_file() or path.is_symlink(): + continue + try: + with path.open("rb") as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + except OSError: + return None + if len(raw) > MAX_MANIFEST_BYTES: + return None + try: + payload = json.loads(raw) + except ValueError: + return None + return payload if isinstance(payload, Mapping) else None + return None + + +def _completion_claim(report: Mapping[str, Any]) -> bool | None: + """``True`` finished, ``False`` denied it, ``None`` said nothing either way.""" + claim: bool | None = None + for name in _COMPLETION_FLAGS: + value = report.get(name) + if value is True: + claim = True + elif value is False: + return False + for field in _COMPLETION_STATUS_FIELDS: + value = report.get(field) + if not isinstance(value, str): + continue + if value.strip().casefold() in _COMPLETION_STATUS_VALUES: + claim = True + else: + # A status the adapter does not recognize is not a completion. + return False + return claim + + +def _indexed_count(report: Mapping[str, Any]) -> int | None: + """How many files the provider says it indexed, if it says at all.""" + for counter in _INDEXED_COUNTERS: + value = report.get(counter) + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + return value + return None + + +def _read_completeness(report: Mapping[str, Any] | None) -> IndexResult: + """Classify a provider run from its own report, defaulting to partial. + + Absent or unreadable evidence is *not* evidence of a complete build, and + neither is a readable report that says nothing. ``complete`` is reached + only by a report shaped the way this adapter understands one: an + affirmative completion claim, a count of what was indexed, and no counter + admitting work left over. An empty object, an unrecognized schema, and a + document that happens to parse all stay ``partial``, which + ``graph_status`` refuses by default. The provider owns no provenance (the + evaluation records this as the first product constraint), so that refusal + is the failure an operator can act on; silently calling it complete is the + one they cannot. + """ + if report is None: + return IndexResult(completeness=PARTIAL, notes=("provider left no readable completion report",)) + notes: list[str] = [] + for counter in _INCOMPLETE_COUNTERS: + value = report.get(counter) + if value is True: + notes.append(f"provider reported {counter}") + elif isinstance(value, int) and not isinstance(value, bool) and value > 0: + notes.append(f"provider reported {value} {counter}") + claim = _completion_claim(report) + if claim is False: + notes.append("provider did not report the extraction as complete") + elif claim is None: + notes.append("provider report carried no completion claim") + indexed = _indexed_count(report) + if indexed is None: + notes.append("provider report did not say how many files it indexed") + if notes: + return IndexResult(completeness=PARTIAL, indexed_files=indexed or 0, notes=tuple(notes)) + return IndexResult(completeness=COMPLETE, indexed_files=indexed) + + +class _BoundedBuffer(io.BytesIO): + """A buffer that refuses to grow past the artifact budget. + + The budget has to be enforced on what this process allocates, as it is + allocated. A check on the finished archive is a check made after the + memory was already taken, and a check on the sum of the file sizes is not + a bound on the archive at all: headers, padding and extended pathname + records are bytes the provider can make this process hold without ever + writing content. + """ + + def write(self, data) -> int: # type: ignore[override] + if self.tell() + len(data) > MAX_ARTIFACT_BYTES: + raise ContextError("local graph artifact exceeds its budget; no generation was published") + return super().write(data) + + +def _pack_state(state_directory: Path) -> bytes: + """Collect the provider's state into one reproducible artifact. + + Names sorted, timestamps and ownership fixed, modes normalized: two builds + of the same commit must produce the same bytes, because the manifest binds + a digest of them. Only regular files are taken -- a symlink in provider + state would name a target outside the artifact, which an immutable + generation cannot carry. + + Both the number of entries and the serialized size are bounded while the + archive is being built, so a provider that wrote pathologically many files + is refused before this process has held them: even collecting the names to + sort them is done against the entry bound rather than into an unbounded + list. + """ + entries: list[Path] = [] + for path in state_directory.rglob("*"): + if path.is_symlink() or not path.is_file(): + continue + entries.append(path) + if len(entries) > MAX_ARTIFACT_ENTRIES: + raise ContextError( + "local graph artifact holds more files than its budget allows; no generation was published" + ) + buffer = _BoundedBuffer() + with tarfile.open(fileobj=buffer, mode="w", format=tarfile.PAX_FORMAT) as archive: + for path in sorted(entries, key=lambda item: str(item.relative_to(state_directory))): + info = tarfile.TarInfo(str(path.relative_to(state_directory))) + info.size = path.stat().st_size + info.mtime = 0 + info.mode = 0o600 + info.uid = info.gid = 0 + info.uname = info.gname = "" + with path.open("rb") as stream: + archive.addfile(info, stream) + return buffer.getvalue() + + +#: How long one extraction may run. A bound on the wall clock a build can cost, +#: not a guess at how long a real one takes. +EXTRACTION_TIMEOUT_SECONDS = 900 + +#: How long a timed-out provider group is given to exit on its own terms before +#: the group is killed outright. Short: the run has already exceeded its whole +#: time budget, and the caller is about to delete the directory these processes +#: are writing into. +_TERMINATION_GRACE_SECONDS = 5.0 + +#: How long to wait for a killed process to be reaped. ``SIGKILL`` is not +#: refusable, so this bounds a wait on the kernel rather than on the child. +_REAP_TIMEOUT_SECONDS = 10.0 + + +#: How often the grace period is re-checked when what is being waited for is +#: the group rather than the direct child. After a normal exit the leader has +#: already been reaped, so there is no child left to wait on and the only way +#: to see the group empty is to ask. +_GROUP_POLL_SECONDS = 0.05 + + +def _signal_group(group: int, number: int) -> None: + try: + os.killpg(group, number) + except OSError: + # Already gone, or never ours to signal. Either way there is nothing + # left to stop, and a failure here must not mask the timeout. + pass + + +def _session_group(child: subprocess.Popen[bytes]) -> int | None: + """The group the child leads, read while the child is still unreaped. + + Read once at launch and retained for the rest of the run, because a pid is + only a safe thing to look a group up from while its process has not been + reaped: afterwards ``os.getpgid`` either fails or answers for whichever + process inherited the number. The group id itself stays safe to signal for + exactly as long as it is worth signalling, because the kernel does not + reuse a pid while it still names a process group with members in it. + + ``None`` means there is no group of this run's own to signal -- the child + never reached one, so the only thing that can be stopped is the child. + """ + try: + group = os.getpgid(child.pid) + except OSError: + return None + if group == os.getpgid(0): + return None + return group + + +def _group_is_empty(group: int) -> bool: + """Whether anything is left in ``group``. + + Signal ``0`` runs the kernel's existence and permission checks without + delivering anything, so this is the group's own answer rather than an + inference from what the leader did. A refusal is not emptiness: something + has to be there for the kernel to refuse on behalf of. + """ + try: + os.killpg(group, 0) + except ProcessLookupError: + return True + except OSError: + return False + return False + + +def _await_group_exit(group: int, timeout: float) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _group_is_empty(group): + return + time.sleep(_GROUP_POLL_SECONDS) + + +def _terminate_process_group(child: subprocess.Popen[bytes], group: int | None) -> None: + """Stop everything the provider started, however this run ended. + + ``subprocess.run``'s own timeout kills the immediate child only. A provider + that forks workers -- and under a launcher such as ``sandbox-exec`` the + process that is signalled may be the launcher rather than the indexer -- + would leave those workers running, still holding CPU and still writing into + a scratch directory the build is about to delete. The child leads its own + session, so one signal to its group reaches every descendant that has not + deliberately left it. + + A leader that exits on its own is not evidence that its workers did. The + group is therefore checked on an ordinary return too, and not only on the + timeout and the interrupt: otherwise a provider that returns while a worker + is still writing leaves ``subprocess_indexer`` packing an artifact that is + concurrently being modified, and ``build_graph`` removing a scratch + directory that is still in use. Checking costs nothing in the ordinary + case, where the group is already empty and nothing is signalled or waited + for. + """ + leader_running = child.poll() is None + if group is None: + if leader_running: + child.kill() + _reap(child) + return + if not leader_running and _group_is_empty(group): + # The ordinary ending: the provider exited and took its workers with it. + return + _signal_group(group, signal.SIGTERM) + if leader_running: + try: + child.wait(timeout=_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + pass + else: + # Nothing here is this process's child any more -- the survivors were + # reparented when the leader died -- so the grace period is spent + # watching the group instead of waiting on a handle. + _await_group_exit(group, _TERMINATION_GRACE_SECONDS) + # Unconditionally, and after the grace period: a leader exiting says nothing + # about workers it started, and this is the last moment anything can stop + # them. + _signal_group(group, signal.SIGKILL) + if leader_running: + _reap(child) + # Sending the signal is not the same as the group being gone, and the + # caller's next act is to pack or delete the state these processes are + # writing. ``SIGKILL`` is not refusable, so this waits on the kernel rather + # than on a cooperating child -- but a process stuck in uninterruptible + # sleep can still outlive it, and a group that cannot be established as + # empty fails the build instead of being assumed gone. + _await_group_exit(group, _REAP_TIMEOUT_SECONDS) + if not _group_is_empty(group): + raise ContextError( + "local graph provider left processes running that could not be stopped; " + "no generation was published" + ) + + +def _reap(child: subprocess.Popen[bytes]) -> None: + try: + child.wait(timeout=_REAP_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: # pragma: no cover - a killed child is reapable + pass + + +def _run_contained( + argv: Sequence[str], + *, + environment: Mapping[str, str], + cwd: str, + timeout: float, +) -> int: + """Run one child in its own process group, stopping the group on any exit. + + Raises :class:`subprocess.TimeoutExpired` once the group has been stopped, + so a caller reports the timeout only after there is nothing left running. + A returned exit status carries the same guarantee: the caller reads the + provider's own status, and by then nothing the provider started is still + running against the state the caller is about to pack up or delete. + """ + with subprocess.Popen( # noqa: S603 - argv is a resolved executable and validated options + list(argv), + # Neither stream is read, and neither may be buffered: a provider that + # logs its progress would otherwise accumulate unbounded output in this + # process for up to the timeout, outside both the tracked-content and + # artifact budgets. The streams are discarded at the kernel rather than + # inherited, because provider diagnostics can echo indexed source and + # this process may be writing a machine-readable report. ``stdin`` goes + # the same way: the child has no operator to prompt. + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=dict(environment), + cwd=cwd, + # A new session, so the child leads a process group that can be + # signalled as a unit and that this process is not a member of. Safe + # only because no stream is inherited: nothing here needs a controlling + # terminal. + start_new_session=True, + ) as child: + # Read before the wait, because the wait may reap the leader and a + # reaped leader's pid is no longer a safe thing to look a group up from. + group = _session_group(child) + try: + return child.wait(timeout=timeout) + finally: + # Every way out of this wait, including returning. An operator's + # Ctrl-C raises ``KeyboardInterrupt`` here, and the provider does + # not see that signal: it leads its own session, so the terminal's + # SIGINT never reaches it. ``Popen.__exit__`` would then wait for a + # child nobody has asked to stop, while ``build_graph`` deletes the + # scratch directory underneath it. A provider that simply exits can + # leave workers behind the same way, so the ordinary return is + # cleaned up on the same path rather than trusted. Unwinding -- + # or returning -- leaves nothing running against state that is + # about to be read, packed, or removed. + _terminate_process_group(child, group) + + +def subprocess_indexer( + executable: str, *, repository: Path, pin: GraphifyPin +) -> Callable[[IndexRequest], IndexResult]: + """Run a pinned provider CLI over the materialized copy, without a network. + + Kept as a factory so the lifecycle never imports or requires a graph + package: a deployment that has installed the pin supplies the executable, + and everything else -- including every test in this repository -- injects + its own callable. The child sees only ``request.environment``, and it sees + it from inside a sandbox that denies it sockets. Resolving the executable, + checking it against the pin, and resolving the sandbox here, rather than at + build time, means an unusable provider, an install that is not the pinned + release, or an uncontainable host fails before a single blob is + materialized. + + ``pin`` is taken here rather than only from each request because it is half + of what the executable *is*: an adapter built for one release must not be a + callable that would run whatever a later request's pin happened to name. + + The argv is the interface the adopt decision evaluated, not a guess at a + conventional one: ``extract`` with the required restrictions and then the + pinned options, in the materialized copy. Everything the provider leaves + behind is then collected and classified from its own report. + """ + command = _resolved_executable(executable) + if containment_mechanism() is None: + raise ContextError( + "local graph builds need an OS sandbox that denies the provider the network and " + "the host filesystem; this host offers none that could be verified" + ) + _verify_provider_installation(command, pin=pin) + # The checkout root, for the same reason the state identity uses it, and + # here it is load-bearing rather than cosmetic: both boundaries below refuse + # what lives *inside the checkout*, and a subdirectory would narrow that + # refusal to part of one. A provider at ``/provider-venv`` must be + # refused for a build run from ``/src`` exactly as it is from the + # root, and the exposure a run is confined to is checked the same way. + repository = checkout_root(repository) + runtime = _provider_read_paths(command, repository=repository) + + def run(request: IndexRequest) -> IndexResult: + if request.pin != pin: + # The verification above spoke for one pin; the launch below reads + # its options and the manifest records it from another. One caller + # passes both, so a divergence is a miswiring rather than an + # operator's doing -- and it would publish a manifest naming a pin + # nothing was checked against, which is the defect this check was + # added to close. + raise ContextError( + "this build's provider pin is not the pin the installed provider was checked " + "against; no generation was published" + ) + _refuse_pre_existing_provider_state(request.source_root) + # Built per run, because the boundary is a function of what this build + # exposes: the materialized copy and the build's own scratch areas are + # writable, the pinned provider's install is readable, and nothing else + # on this host is in the child's filesystem view at all. + sandbox = containment_prefix( + writable=(request.source_root, *request.writable), + readable=runtime, + repository=repository, + ) + # Normalized again at the point of launch, not because the pin could + # arrive without the restrictions -- it cannot -- but because this is + # the line that decides what the provider is actually asked to do, and + # it should be readable here without trusting a constructor elsewhere. + options = _extraction_options(request.pin.options) + try: + returncode = _run_contained( + [*sandbox, command, _PROVIDER_EXTRACT, *options], + environment=request.environment, + cwd=str(request.source_root), + timeout=EXTRACTION_TIMEOUT_SECONDS, + ) + except subprocess.TimeoutExpired: + # Raised only once the whole process group has been stopped, so the + # caller may delete the scratch directory without racing a worker. + raise ContextError( + "local graph provider exceeded its time budget; no generation was published" + ) from None + except (OSError, subprocess.SubprocessError): + raise ContextError("local graph provider could not be run from its pinned install") from None + if returncode != 0: + raise ContextError("local graph provider failed; no generation was published") + state_directory = _provider_state_directory(request.source_root) + result = _read_completeness(_provider_report(state_directory)) + _write_private_file(request.output_path, _pack_state(state_directory)) + return result + + return run + + +@dataclass(frozen=True) +class BuildManifest: + """The immutable record bound to one published generation. + + Every field an artifact must carry under issue #913 lives here: the full + commit and tree it was built from, the provider pin and options that built + it, when, the tracked census it was allowed to see and that census's + digest, the graph's own digest and byte count, and whether the provider + considered the result complete. + """ + + generation: str + schema: str + commit: str + tree: str + provider: dict[str, Any] + built_at: str + tracked_files: int + tracked_bytes: int + census_digest: str + graph_digest: str + graph_bytes: int + completeness: str + skipped_paths: int + indexed_files: int + + def to_json(self) -> dict[str, Any]: + return { + "schema": self.schema, + "generation": self.generation, + "commit": self.commit, + "tree": self.tree, + "provider": self.provider, + "built_at": self.built_at, + "tracked_files": self.tracked_files, + "tracked_bytes": self.tracked_bytes, + "census_digest": self.census_digest, + "graph_digest": self.graph_digest, + "graph_bytes": self.graph_bytes, + "completeness": self.completeness, + "skipped_paths": self.skipped_paths, + "indexed_files": self.indexed_files, + } + + def shareable_summary(self) -> dict[str, Any]: + """Metadata only: counts, digests and revision names, never content.""" + return { + "schema": "code_mower.contextGraphBuildSummary.v1", + "generation": self.generation, + "commit": self.commit, + "tree": self.tree, + "provider_version": self.provider.get("version"), + "built_at": self.built_at, + "tracked_files": self.tracked_files, + "census_digest": self.census_digest, + "graph_digest": self.graph_digest, + "graph_bytes": self.graph_bytes, + "completeness": self.completeness, + } + + +def load_manifest(payload: Mapping[str, Any]) -> BuildManifest: + """Validate a manifest. Every unreadable shape is a refusal, not a default.""" + if not isinstance(payload, Mapping): + raise ContextError("local graph manifest must be an object") + expected = { + "schema", "generation", "commit", "tree", "provider", "built_at", + "tracked_files", "tracked_bytes", "census_digest", "graph_digest", + "graph_bytes", "completeness", "skipped_paths", "indexed_files", + } + if set(payload) != expected: + raise ContextError("local graph manifest fields are missing or unrecognized") + if payload["schema"] != MANIFEST_SCHEMA: + raise ContextError("unsupported local graph manifest schema") + generation = payload["generation"] + if not isinstance(generation, str) or not _GENERATION.fullmatch(generation): + raise ContextError("local graph generation must be an opaque identifier") + completeness = payload["completeness"] + if completeness not in (COMPLETE, PARTIAL): + raise ContextError("unsupported local graph completeness") + provider = payload["provider"] + pin = load_pin(provider) if isinstance(provider, Mapping) else None + if pin is None: + raise ContextError("local graph manifest must record its provider pin") + _timestamp(payload["built_at"]) + return BuildManifest( + generation=generation, + schema=MANIFEST_SCHEMA, + commit=_object_name(payload["commit"]), + tree=_object_name(payload["tree"]), + provider=pin.as_metadata(), + built_at=payload["built_at"], + tracked_files=_size(payload["tracked_files"], MAX_TRACKED_FILES), + tracked_bytes=_size(payload["tracked_bytes"], MAX_TRACKED_BYTES), + census_digest=_digest(payload["census_digest"]), + graph_digest=_digest(payload["graph_digest"]), + graph_bytes=_size(payload["graph_bytes"], MAX_ARTIFACT_BYTES), + completeness=completeness, + skipped_paths=_size(payload["skipped_paths"], MAX_SKIPPED_PATHS), + indexed_files=_size(payload["indexed_files"], MAX_TRACKED_FILES), + ) + + +def checkout_root(repository: Path) -> Path: + """The worktree root that owns ``repository``, or the path itself. + + State identity is per checkout, not per directory. A census reads the + commit's whole tree -- ``ls-tree --full-tree``, never the invocation + directory's slice of it -- so ``status`` run in ``src/`` asks about exactly + the generation ``status`` run at the root published, and must resolve to + it. Deriving the identity from the invocation directory instead made every + subdirectory its own workspace: a graph built at the root reported + ``absent`` from ``src/``, and ``remove`` from there deleted nothing while + reporting success. + + Git answers this per worktree, which is what keeps linked worktrees + separate: each reports its own root, and each may hold a different + revision, so they must not share a generation. A path Git cannot place -- + not a repository, a bare one, or no Git on the host -- keeps the resolved + path it was given, which is what this derived before. Nothing is refused + here: the verbs that need Git already fail on their own terms, and a + lifecycle that could not even name its state would fail worse. + """ + resolved = Path(repository).resolve() + try: + toplevel = _git(resolved, "rev-parse", "--show-toplevel", permit_failure=True).strip() + except ContextError: + return resolved + if not toplevel: + return resolved + # Resolved the same way the fallback is, so one checkout has one identity + # however it was spelled -- and so a root that was already canonical keeps + # the workspace name its existing generations are filed under. + return Path(os.path.realpath(toplevel)) + + +def workspace_id(repository: Path) -> str: + """A stable private name for one checkout. + + Derived from the resolved path so two worktrees of the same repository get + separate state and can never read each other's generations, and hashed so + the operator's directory layout is not spelled out in a shared location. + + The path must already be a checkout root. ``GraphStateRoot`` is the one + place identity is derived, and it normalizes through ``checkout_root`` + first; a caller that hashes a subdirectory gets a name nothing else uses. + """ + return hashlib.sha256(str(Path(repository).resolve()).encode()).hexdigest()[:32] + + +#: What a missing component means to a walk: create it, stop there, or refuse. +_MISSING_CREATE = "create" +_MISSING_STOP = "stop" +_MISSING_REFUSE = "refuse" + + +def _open_private_at(parent: int | None, name: str, *, missing: str, private: bool = True) -> int | None: + """Open one directory *relative to a descriptor*, following nothing. + + ``parent`` is the descriptor the name is resolved against, so the kernel + resolves exactly one component and ``O_NOFOLLOW`` covers all of it. That is + the difference between checking a path and traversing one: a path opened by + its full spelling is re-resolved from the root every time, and any ancestor + may have become a symlink since it was last looked at. + + ``mkdir`` runs against the same descriptor for the same reason. Creating + with ``parents=True`` from a full path would follow an ancestor that became + a symlink between the check and the creation -- the state-root defect this + replaces -- and no later check on the leaf can see that it happened. + """ + unsafe = "local graph state directory is unavailable or unsafe" + try: + handle = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + except FileNotFoundError: + if missing == _MISSING_STOP: + return None + if missing != _MISSING_CREATE: + raise ContextError(unsafe) from None + # Opened before it is created, rather than creating unconditionally and + # reading the errno: this walk now traverses the whole absolute base, + # including ancestors like ``/usr`` that exist and that nobody may + # write. Whether such a ``mkdir`` reports ``EEXIST`` or ``EACCES`` + # first is the kernel's business, and a boundary should not rest on it. + try: + os.mkdir(name, mode=0o700, dir_fd=parent) + except FileExistsError: + pass + except OSError: + raise ContextError(unsafe) from None + try: + handle = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + except OSError: + # Created as a directory and already something else, or something + # else won the race: either way this is not a directory this walk + # may descend. + raise ContextError(unsafe) from None + except OSError: + # ``ELOOP`` lands here: the component is a symlink, and a symlink is + # not a directory this class owns however private its target may be. + raise ContextError(unsafe) from None + if not private: + # An ancestor *above* the operator's private root: this class does not + # own its mode and must not judge it. What matters there is only that + # it was traversed as a directory rather than through a link. + return handle + try: + _private(handle, directory=True) + except ContextError: + os.close(handle) + raise + return handle + + +class GraphStateRoot: + """Private, operator-owned, 0700 state for one checkout's generations. + + Layout under ``/graph//``:: + + generations//manifest.json + generations//graph.bin + current -- the published generation's name + + The directory is created 0700 and every read re-checks ownership and mode, + so state that was later loosened -- by an umask change, a restore, or a + careless ``chmod -R`` -- fails closed instead of being used. + """ + + def __init__(self, repository: Path, *, root: Path | None = None): + #: The checkout root, not the directory the command was run from. Every + #: verb reaches its state through this class, so normalizing here is + #: what makes ``build`` at the root and ``status`` in ``src/`` name one + #: workspace; callers read it back to run Git against the same root the + #: identity came from. + self.repository = checkout_root(repository) + base = Path(root) if root is not None else default_context_root() + if not base.is_absolute(): + raise ContextError("local graph state requires an absolute private directory") + # Canonicalized once, here, and never spelled lexically again: every + # later open, mkdir, lock, and removal travels the path that + # ``_refuse_state_inside_a_repository`` checked. Checking a resolved + # snapshot and then writing through the original spelling would leave a + # symlinked ancestor free to be retargeted in between -- the check + # passes against one directory and the writes land in another. Ordinary + # private roots do have symlinked ancestors (macOS puts ``/tmp`` behind + # a link into its ``private`` directory), so they are resolved rather + # than refused. + self.base = Path(os.path.realpath(base)) + self.workspace = workspace_id(self.repository) + self.path = self.base / "graph" / self.workspace + #: The device and inode the base was first observed as, established by + #: the no-follow walk and re-checked by every later one. Canonicalizing + #: at construction settles what the ancestors mean *now*; this is what + #: notices that they stopped meaning it. + self._identity: tuple[int, int] | None = None + + @property + def generations_path(self) -> Path: + return self.path / "generations" + + @property + def lock_path(self) -> Path: + """Beside the state directory, deliberately not inside it. + + ``remove`` deletes the whole tree while holding this lock. A lock file + inside that tree would be unlinked mid-removal, and the next builder + would create a *new* inode and acquire a lock nobody else is holding -- + two processes, two files, no mutual exclusion. Keeping it one level up + means the inode a holder waits on is the inode the remover holds. + """ + return self.path.parent / f"{self.workspace}.lock" + + #: The components this class owns below the canonical base, outermost + #: first. Spelled out one at a time because each is created and opened + #: against its parent's descriptor: ``mkdir(parents=True)`` would both + #: apply ``0o700`` to the leaf only -- leaving intermediates at the process + #: umask -- and follow an ancestor that became a symlink in between. + @property + def _components(self) -> tuple[str, ...]: + return ("graph", self.workspace, "generations") + + def _open_base(self, *, missing: str) -> int | None: + """Open the private root by walking it from ``/``, following nothing. + + A single ``open(base, O_NOFOLLOW)`` is not this, and the difference is + the whole of the defect it replaces. ``O_NOFOLLOW`` refuses only the + *final* component; every ancestor above it is resolved by the kernel + exactly as a symlink planted there would want. So a base whose ancestor + was replaced by a link after construction passes the leaf check -- + because the leaf really is a directory and really is not a link. It is + simply not the directory that was checked. Finding the deepest existing + prefix first does not help: that prefix is still opened by its full + absolute spelling, in one call, through whatever its ancestors have + become. + + Every component is opened against its parent's descriptor instead, so + the kernel resolves exactly one name at a time and ``O_NOFOLLOW`` + covers all of it. The base was canonicalized in ``__init__``, so on an + untampered host no component of it is a link and this walk is a + restatement of the path; a component that has become one since is + precisely what must fail, whether or not the components below it exist + already. + + Components above the operator's root are traversed but not judged for + ownership or mode -- that is not this class's to own. What matters + there is only that each was a real directory rather than a link. + """ + parts = self.base.parts + # The root is not a component anybody can replace, and mkdir on it is + # meaningless; it is opened, never created. + handle = _open_private_at(None, parts[0], missing=_MISSING_REFUSE, private=False) + if handle is None: # pragma: no cover - _MISSING_REFUSE raises instead + return None + try: + for name in parts[1:]: + deeper = _open_private_at(handle, name, missing=missing, private=False) + if deeper is None: + os.close(handle) + return None + os.close(handle) + handle = deeper + except BaseException: + os.close(handle) + raise + info = os.fstat(handle) + identity = (info.st_dev, info.st_ino) + if self._identity is None: + self._identity = identity + elif self._identity != identity: + # The walk was clean and still arrived somewhere else than it did + # last time: an ancestor was swapped between two operations on the + # same root. Refuse rather than carry on against a directory this + # instance never checked. + os.close(handle) + raise ContextError("local graph state directory is unavailable or unsafe") + return handle + + def _open_owned(self, *, depth: int) -> int | None: + """The descriptor for one owned directory, or ``None`` if it is absent. + + This is what every mutation below holds instead of a path. An earlier + revision revalidated the base -- compared the walk's inode against the + one the full spelling resolved to -- immediately before each rename and + removal, which reads as safe and is not: the check and the use are two + separate resolutions of the same spelling, and an ancestor swapped + between them lands the use somewhere the check never saw. Rechecking a + path cannot close that race; not resolving the path a second time is + what closes it. + + Unlike ``_walk`` this refuses to return a *shallower* directory when a + component is missing: a caller asking for the generations directory + must not silently receive the workspace directory and mutate it. + """ + handle = self._open_base(missing=_MISSING_STOP) + if handle is None: + return None + try: + for component in self._components[:depth]: + deeper = _open_private_at(handle, component, missing=_MISSING_STOP) + if deeper is None: + os.close(handle) + return None + os.close(handle) + handle = deeper + except BaseException: + os.close(handle) + raise + return handle + + def _walk(self, *, depth: int, missing: str) -> int: + """Descend the owned components from the base, one descriptor at a time. + + Returns the deepest descriptor reached; the caller closes it. With + ``missing=_MISSING_STOP`` a component that does not exist ends the walk + rather than failing it, which is what a read of state that was never + built needs. Nothing below a component that failed its privacy check is + ever opened, because there is no descriptor left to open it against. + """ + opened = self._open_base(missing=missing) + if opened is None: + return -1 + handle = opened + try: + for component in self._components[:depth]: + deeper = _open_private_at(handle, component, missing=missing) + if deeper is None: + return handle + os.close(handle) + handle = deeper + except BaseException: + os.close(handle) + raise + return handle + + def _close_walk(self, handle: int) -> None: + if handle >= 0: + os.close(handle) + + def verify_private(self, *, create: bool = False) -> None: + """Re-check ownership and mode on every directory this class owns. + + Called on every read, not only at creation: state that was loosened + after the fact -- by a umask change, a restore, or a careless recursive + chmod -- must fail closed rather than be trusted because it was private + when it was written. + """ + missing = _MISSING_CREATE if create else _MISSING_STOP + self._close_walk(self._walk(depth=len(self._components), missing=missing)) + + def ensure(self) -> None: + """Create the private tree, refusing to place state inside a repository.""" + self._refuse_state_inside_a_repository() + self.verify_private(create=True) + + def _refuse_state_inside_a_repository(self) -> None: + """No generation may be written inside a repository, by any spelling. + + A lexical walk alone reads ``--state-dir /outside/link/state`` as being + outside every repository even when ``/outside/link`` points at + ``/repo/subdir``. The path walked here is the canonical one built in + ``__init__``, so a symlinked ancestor that exists now is resolved before + it is judged. + + An ancestor that does *not* exist yet cannot be resolved by anybody, and + this check alone would miss a component created as a symlink afterwards. + That case is answered by construction rather than by re-checking: every + component below the base is created and opened against its parent's + descriptor with ``O_NOFOLLOW``, so a component that is a symlink when + the build reaches it is refused outright instead of traversed. The two + together leave no window: what exists is resolved, and what does not + exist yet can only be created here, by this process, as a real + directory. + """ + if any((parent / ".git").exists() for parent in (self.path, *self.path.parents)): + raise ContextError("local graph state must stay outside Git repositories") + + def _ensure_lock_directory(self) -> int: + """Create only what the lock file needs, not the generations tree. + + ``remove`` takes the same lock, and a removal that first created the + state it was asked to delete would report success for a tree it made + itself. Returns the descriptor of the directory the lock file lives in, + so the lock is opened relative to the directory that was just checked + rather than re-resolved from the root. + """ + self._refuse_state_inside_a_repository() + return self._walk(depth=1, missing=_MISSING_CREATE) + + def lock(self): + """Serialize builds *and removals* for one checkout. + + Concurrent builds would race publish; a removal running beside a build + would delete the sources, output, and generations out from under it. + Both take this lock, so the whole set of lifecycle operations that + mutate state for one checkout is serialized rather than just the pair + that was obviously racy. + """ + parent = self._ensure_lock_directory() + try: + handle = os.open( + self.lock_path.name, + os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, + 0o600, + dir_fd=parent, + ) + finally: + # The lock file is opened against the descriptor the walk verified, + # so the directory it lands in is the directory that was checked + # and not whatever that path spells by the time this line runs. + self._close_walk(parent) + stream = os.fdopen(handle, "a+", encoding="utf-8") + try: + _private(stream.fileno()) + return _BuildLock(stream) + except ContextError: + stream.close() + raise + + def current_generation(self) -> str | None: + pointer = self.path / CURRENT_NAME + try: + handle = os.open(pointer, os.O_RDONLY | os.O_NOFOLLOW) + except FileNotFoundError: + return None + except OSError: + raise ContextError("local graph state directory is unavailable or unsafe") from None + try: + _private(handle) + with os.fdopen(handle, "rb", closefd=False) as stream: + name = stream.read(64).decode("utf-8", "replace").strip() + finally: + os.close(handle) + if not _GENERATION.fullmatch(name): + raise ContextError("local graph generation pointer is corrupt; rebuild the graph") + return name + + def generation_names(self) -> list[str]: + if not self.path.exists(): + return [] + self.verify_private() + try: + names = os.listdir(self.generations_path) + except FileNotFoundError: + return [] + except OSError: + raise ContextError("local graph state directory is unavailable or unsafe") from None + return sorted(name for name in names if _GENERATION.fullmatch(name)) + + def read_manifest(self, generation: str) -> BuildManifest: + """Read and validate one generation's manifest, permissions included.""" + if not _GENERATION.fullmatch(generation): + raise ContextError("local graph generation must be an opaque identifier") + directory = self.generations_path / generation + handle = _open_private_at(None, str(directory), missing=_MISSING_REFUSE) + if handle is not None: # _MISSING_REFUSE raises rather than returning None + os.close(handle) + try: + handle = os.open(directory / MANIFEST_NAME, os.O_RDONLY | os.O_NOFOLLOW) + except OSError: + raise ContextError("local graph generation is missing its manifest") from None + try: + _private(handle) + with os.fdopen(handle, "rb", closefd=False) as stream: + raw = stream.read(MAX_MANIFEST_BYTES + 1) + finally: + os.close(handle) + if len(raw) > MAX_MANIFEST_BYTES: + raise ContextError("local graph manifest exceeds its bound") + try: + payload = json.loads(raw) + except ValueError: + raise ContextError("local graph manifest is corrupt; rebuild the graph") from None + manifest = load_manifest(payload) + if manifest.generation != generation: + raise ContextError("local graph manifest does not match its generation") + return manifest + + def artifact_path(self, generation: str) -> Path: + if not _GENERATION.fullmatch(generation): + raise ContextError("local graph generation must be an opaque identifier") + return self.generations_path / generation / ARTIFACT_NAME + + def publish(self, manifest: BuildManifest, artifact: bytes) -> BuildManifest: + """Assemble a generation under a staging name, then rename it into place. + + Both steps are atomic renames, in an order a reader can survive: the + generation directory becomes visible whole, and only then does + ``current`` start naming it. A crash between the two leaves an + unreferenced generation, which ``prune`` removes; it never leaves a + pointer to a directory that does not exist. + + Nothing is written until the serialized manifest has been read back + through ``load_manifest`` and measured against the bound a reader + applies. A manifest this process can write but no reader can load would + otherwise publish, move ``current`` onto it, prune the previous usable + generation, and read back ``invalid`` on the next status: a build that + reports success while destroying the only generation that worked. The + check belongs here, at the one boundary every generation crosses, + rather than at each of the places that fill a single field in. + """ + serialized = json.dumps( + manifest.to_json(), allow_nan=False, sort_keys=True, separators=(",", ":") + ).encode() + if len(serialized) > MAX_MANIFEST_BYTES: + raise ContextError("local graph manifest exceeds its bound; no generation was published") + if load_manifest(json.loads(serialized)).generation != manifest.generation: + raise ContextError("local graph manifest does not match its generation") + self.ensure() + # Every step below runs against a descriptor the no-follow walk opened, + # never against a path: ``os.rename`` with ``src_dir_fd``/``dst_dir_fd`` + # resolves one component on each side, so an ancestor that is swapped + # after the walk has nothing left to redirect. + generations = self._walk(depth=len(self._components), missing=_MISSING_REFUSE) + try: + staging = "." + uuid.uuid4().hex + ".staging" + os.mkdir(staging, mode=0o700, dir_fd=generations) + try: + staged = os.open( + staging, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=generations + ) + try: + _write_private_file(ARTIFACT_NAME, artifact, dir_fd=staged) + _write_private_file(MANIFEST_NAME, serialized, dir_fd=staged) + os.fsync(staged) + finally: + os.close(staged) + os.rename( + staging, + manifest.generation, + src_dir_fd=generations, + dst_dir_fd=generations, + ) + os.fsync(generations) + except BaseException: + _remove_tree_at(generations, staging, ignore_errors=True) + raise + finally: + self._close_walk(generations) + state = self._walk(depth=len(self._components) - 1, missing=_MISSING_REFUSE) + try: + temporary = "." + uuid.uuid4().hex + ".tmp" + try: + _write_private_file( + temporary, manifest.generation.encode() + b"\n", dir_fd=state + ) + os.replace(temporary, CURRENT_NAME, src_dir_fd=state, dst_dir_fd=state) + os.fsync(state) + except BaseException: + _unlink_at(state, temporary, ignore_errors=True) + raise + finally: + self._close_walk(state) + return manifest + + def prune(self, *, keep: str | None) -> list[str]: + """Remove every generation but ``keep``, including crashed stagings.""" + removed: list[str] = [] + generations = self._open_owned(depth=len(self._components)) + if generations is None: + return removed + try: + for name in sorted(os.listdir(generations)): + if name == keep: + continue + _remove_tree_at(generations, name, ignore_errors=True) + if _GENERATION.fullmatch(name): + removed.append(name) + os.fsync(generations) + finally: + self._close_walk(generations) + return removed + + def remove_all(self) -> bool: + """Delete this checkout's graph state, serialized against builders. + + Taken under the build lock: without it, a removal can delete a running + build's materialized sources, its output, and the generations + directory, after which the builder either fails or recreates state + that ``remove`` has already reported as gone. + + The lock file itself survives, by design. It is an empty 0600 file + outside the deleted tree that carries no indexed content, and it is the + stable inode the next builder and the next remover agree on. + """ + if not self.path.exists() and not self.lock_path.exists(): + # Nothing exists and no builder can be running: a builder creates + # the lock file before it creates any state, so an absent lock file + # means there is nothing to serialize against. Checked first so a + # removal on a fresh install does not create a private tree merely + # to report that it was empty. + return False + with self.lock(): + if not self.path.exists(): + return False + # Refuse to delete a tree that is not ours; a loosened or foreign + # directory is reported, not recursively removed. + self.verify_private() + holder = self._open_owned(depth=len(self._components) - 2) + if holder is None: # pragma: no cover - verify_private ran first + return False + try: + # The privacy check and the deletion run against *the same* + # descriptor, so a directory swapped in after the check cannot + # become the directory that is deleted. Opening the workspace + # by full path here would reintroduce the whole finding: a + # recursive delete whose root is resolved once more, from ``/``, + # through whatever the ancestors have become by then. + owned = _open_private_at(holder, self.workspace, missing=_MISSING_STOP) + if owned is None: # pragma: no cover - existence checked above + return False + try: + for entry in os.listdir(owned): + _remove_tree_at(owned, entry) + os.fsync(owned) + finally: + os.close(owned) + os.rmdir(self.workspace, dir_fd=holder) + os.fsync(holder) + finally: + self._close_walk(holder) + return True + + +class _BuildLock: + def __init__(self, stream): + self._stream = stream + self._guard = None + + def __enter__(self): + self._guard = exclusive_handle_lock(self._stream, timeout_seconds=35) + try: + self._guard.__enter__() + except FileLockError: + raise ContextError("a local graph build is already running for this checkout") from None + return self + + def __exit__(self, *exception): + try: + if self._guard is not None: + self._guard.__exit__(*exception) + finally: + self._stream.close() + return False + + +def _write_private_file(path: Path | str, payload: bytes, *, dir_fd: int | None = None) -> None: + """Create one 0600 file, optionally relative to an already-verified directory. + + With ``dir_fd`` the name must be a single component: the kernel resolves it + against that descriptor and nothing above it is re-resolved, so an ancestor + that becomes a symlink cannot redirect the write. + """ + handle = os.open( + path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=dir_fd + ) + with os.fdopen(handle, "wb", closefd=False) as stream: + stream.write(payload) + stream.flush() + os.fsync(handle) + os.close(handle) + + +def _unlink_at(parent: int, name: str, *, ignore_errors: bool = False) -> None: + """Unlink one component against its parent's descriptor, following nothing.""" + try: + os.unlink(name, dir_fd=parent) + except FileNotFoundError: + pass + except OSError: + if not ignore_errors: + raise ContextError("local graph state directory is unavailable or unsafe") from None + + +def _remove_tree_at(parent: int, name: str, *, ignore_errors: bool = False) -> None: + """Delete a tree without ever re-resolving a path from the root. + + ``shutil.rmtree`` cannot be used for this. However carefully it walks + *below* its argument, the argument itself is a full path that the kernel + resolves from ``/`` at the moment the call is entered -- so an ancestor + swapped between the check and that instant sends the whole recursive + deletion somewhere else. Re-checking the path first does not help: the + check and the use are two resolutions of the same spelling, and the race + lives exactly between them. + + Here every descent opens one component against its parent's descriptor with + ``O_NOFOLLOW``, and every unlink names one component against the descriptor + of the directory that really holds it. There is no second resolution to + win, so there is no window to win it in. A symlink encountered anywhere in + the tree is unlinked as the link it is and never followed. + """ + try: + handle = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent) + except FileNotFoundError: + return + except OSError: + # ``ENOTDIR`` for an ordinary file, ``ELOOP`` for a symlink: both are + # removed by name against this descriptor rather than descended into. + _unlink_at(parent, name, ignore_errors=ignore_errors) + return + try: + for entry in os.listdir(handle): + _remove_tree_at(handle, entry, ignore_errors=ignore_errors) + os.fsync(handle) + finally: + os.close(handle) + try: + os.rmdir(name, dir_fd=parent) + except FileNotFoundError: + pass + except OSError: + if not ignore_errors: + raise ContextError("local graph state directory is unavailable or unsafe") from None + + +@dataclass(frozen=True) +class GenerationStatus: + """A shareable verdict about the published generation, if any.""" + + state: str + generation: str | None = None + manifest: BuildManifest | None = None + detail: str = "" + + @property + def usable(self) -> bool: + return self.state == "current" + + def shareable_summary(self) -> dict[str, Any]: + summary: dict[str, Any] = { + "schema": "code_mower.contextGraphStatus.v1", + "state": self.state, + "usable": self.usable, + } + if self.detail: + summary["detail"] = self.detail + if self.manifest is not None: + summary["build"] = self.manifest.shareable_summary() + elif self.generation is not None: + summary["generation"] = self.generation + return summary + + +def build_graph( + repository: Path, + *, + pin: GraphifyPin, + indexer: Callable[[IndexRequest], IndexResult], + root: Path | None = None, + revision: str = "HEAD", + now: datetime | None = None, + keep_previous: bool = False, +) -> BuildManifest: + """Materialize one commit, index the copy, and publish a new generation. + + This is the whole lifecycle in one call, and it is the only way a + generation is created. ``refresh`` is the same operation: an explicit + rebuild that publishes a new immutable generation rather than mutating the + one in place, which is why nothing here ever writes into an existing + generation directory. + """ + state = GraphStateRoot(repository, root=root) + # Read the checkout through the root its identity was derived from. The + # census is whole-tree either way, but binding the two together means a + # build from a subdirectory cannot bind one directory's name to another + # directory's Git answers. + repository = state.repository + commit, tree = resolve_revision(repository, revision) + census = read_tracked_census(repository, commit) + with state.lock(): + # The lock only creates what the lock file needs, so that ``remove`` + # can take it without materializing the tree it was asked to delete. + # A build does want the whole private tree, created 0700 at every + # level before anything is written into it. + state.ensure() + build_root = state.path / ("." + uuid.uuid4().hex + ".build") + build_root.mkdir(mode=0o700, parents=True) + try: + source_root = build_root / "source" + home = build_root / "home" + temporary = build_root / "tmp" + for directory in (home, temporary): + directory.mkdir(mode=0o700) + materialize_tracked_files(repository, census, source_root) + output_path = build_root / ARTIFACT_NAME + result = indexer( + IndexRequest( + source_root=source_root, + output_path=output_path, + environment=scrubbed_environment(home=home, temporary=temporary), + pin=pin, + commit=commit, + tree=tree, + # The same two directories the scrubbed environment points + # at, so what the provider is told to use and what it is + # allowed to write are one decision rather than two. + writable=(home, temporary), + ) + ) + if not isinstance(result, IndexResult) or result.completeness not in (COMPLETE, PARTIAL): + raise ContextError("local graph provider returned an unsupported result") + if not output_path.is_file(): + raise ContextError("local graph provider produced no artifact") + # Read to the budget and one byte past it, rather than trusting a + # size taken before the read: the artifact is provider output, and + # the budget has to bound what this process allocates for it. + with output_path.open("rb") as stream: + artifact = stream.read(MAX_ARTIFACT_BYTES + 1) + if len(artifact) > MAX_ARTIFACT_BYTES: + raise ContextError("local graph artifact exceeds its budget; no generation was published") + manifest = BuildManifest( + generation=uuid.uuid4().hex, + schema=MANIFEST_SCHEMA, + commit=commit, + tree=tree, + provider=pin.as_metadata(), + built_at=(now or datetime.now(timezone.utc)).astimezone(timezone.utc).isoformat(), + tracked_files=census.file_count, + tracked_bytes=census.total_bytes, + census_digest=census.digest, + graph_digest=hashlib.sha256(artifact).hexdigest(), + graph_bytes=len(artifact), + completeness=result.completeness, + skipped_paths=len(census.skipped), + indexed_files=min(_size(result.indexed_files, MAX_TRACKED_FILES), census.file_count), + ) + published = state.publish(manifest, artifact) + if not keep_previous: + # Inside the lock, with publication. Pruning after the lock is + # released would let a second builder publish first and then + # have its generation -- or its staging directory -- deleted by + # this one, leaving ``current`` naming a directory that is gone. + state.prune(keep=published.generation) + finally: + shutil.rmtree(build_root, ignore_errors=True) + return published + + +#: How many times a reader will re-read a generation that was replaced under +#: it. Bounded because the only thing that moves the pointer is a publish, and +#: a refresh loop fast enough to outrun three reads is not a state worth +#: blocking on. +_STATUS_ATTEMPTS = 3 + + +def graph_status( + repository: Path, + *, + root: Path | None = None, + revision: str = "HEAD", + require_complete: bool = True, +) -> GenerationStatus: + """Report whether the published generation may be used, and why not if not. + + Every failure mode issue #913 names resolves here to a non-``current`` + state, and every non-``current`` state is unusable. Nothing falls back to a + previous generation: a consumer that cannot have the revision it asked for + is told so rather than handed an older answer that looks fresh. + + Readers take no lock, so a refresh can publish and prune between the moment + this reads the ``current`` pointer and the moment it validates what that + pointer named. The generation is then genuinely gone, and reporting it + ``invalid`` or ``corrupt`` would describe a directory a healthy build had + just superseded rather than anything wrong with the graph. A failing read + is therefore confirmed against the pointer before it is returned, and a + pointer that moved is read again. + """ + state = GraphStateRoot(repository, root=root) + repository = state.repository + for attempt in range(_STATUS_ATTEMPTS): + status = _status_once( + state, repository, revision=revision, require_complete=require_complete + ) + if status.usable or status.generation is None or attempt == _STATUS_ATTEMPTS - 1: + return status + try: + if state.current_generation() == status.generation: + # The pointer still names what was just validated, so the + # verdict is about the operator's graph, not a race. + return status + except ContextError: + return status + return status + + +def _status_once( + state: GraphStateRoot, + repository: Path, + *, + revision: str, + require_complete: bool, +) -> GenerationStatus: + """One validation pass over whichever generation ``current`` names now.""" + try: + if not state.path.exists(): + return GenerationStatus(state="absent", detail="no local graph has been built for this checkout") + state.verify_private() + generation = state.current_generation() + except ContextError as error: + return GenerationStatus(state="invalid", detail=str(error)) + if generation is None: + return GenerationStatus(state="absent", detail="no local graph generation is published") + try: + manifest = state.read_manifest(generation) + except ContextError as error: + return GenerationStatus(state="invalid", generation=generation, detail=str(error)) + try: + artifact_path = state.artifact_path(generation) + handle = os.open(artifact_path, os.O_RDONLY | os.O_NOFOLLOW) + try: + _private(handle) + info = os.fstat(handle) + if info.st_size > MAX_ARTIFACT_BYTES: + # Checked before the manifest comparison and before any read: + # an artifact that grew past its budget is refused without + # being hashed, however plausible its manifest looks. + return GenerationStatus(state="oversized", generation=generation, manifest=manifest, + detail="local graph artifact exceeds its budget") + if info.st_size != manifest.graph_bytes: + return GenerationStatus(state="corrupt", generation=generation, manifest=manifest, + detail="local graph artifact size does not match its manifest") + digest = hashlib.sha256() + with os.fdopen(handle, "rb", closefd=False) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + finally: + os.close(handle) + except ContextError as error: + return GenerationStatus(state="invalid", generation=generation, manifest=manifest, detail=str(error)) + except OSError: + return GenerationStatus(state="corrupt", generation=generation, manifest=manifest, + detail="local graph artifact is unreadable") + if digest.hexdigest() != manifest.graph_digest: + return GenerationStatus(state="corrupt", generation=generation, manifest=manifest, + detail="local graph artifact does not match its recorded digest") + try: + commit, tree = resolve_revision(Path(repository), revision) + except ContextError as error: + return GenerationStatus(state="invalid", generation=generation, manifest=manifest, detail=str(error)) + if (manifest.commit, manifest.tree) != (commit, tree): + return GenerationStatus(state="stale", generation=generation, manifest=manifest, + detail="local graph was built from a different revision; refresh it") + if require_complete and manifest.completeness != COMPLETE: + return GenerationStatus(state="partial", generation=generation, manifest=manifest, + detail="local graph build was incomplete; refresh it") + return GenerationStatus(state="current", generation=generation, manifest=manifest) + + +def remove_graph(repository: Path, *, root: Path | None = None) -> bool: + """Delete this checkout's graph state. Returns whether anything was removed.""" + return GraphStateRoot(repository, root=root).remove_all() + + +def doctor_report( + repository: Path, + *, + pin: GraphifyPin | None, + root: Path | None = None, + revision: str = "HEAD", +) -> dict[str, Any]: + """Posture checks for the local graph: metadata only, never content. + + Reports ``skip`` rather than ``fail`` when no graph is configured or built. + The lifecycle is optional, and an operator who never opted in has nothing + wrong with their installation. + """ + checks: list[dict[str, Any]] = [] + + def record(name: str, status: str, message: str, **extra: Any) -> None: + checks.append({"check": name, "status": status, "message": message, **extra}) + + if pin is None: + record("context-graph-pin", "skip", "no local graph provider is pinned; the lifecycle is optional") + record("context-graph-isolation", "skip", "no provider is pinned, so nothing would be launched") + else: + record("context-graph-pin", "pass", "local graph provider is pinned to one exact release", + requirement=pin.requirement, wheel_sha256=pin.wheel_sha256) + # Named while it is still a posture question. Discovering that this + # host cannot contain a provider is worth knowing before a build + # refuses, and the check reports the mechanism rather than its + # arguments, which would be noise. + mechanism = containment_mechanism() + if mechanism is None: + record("context-graph-isolation", "fail", + "no OS sandbox on this host was observed denying a child process both the " + "network and the host filesystem; builds will refuse") + else: + record("context-graph-isolation", "pass", + "the provider would run inside a sandbox that denies it the network and " + "everything outside the build's own directories", + mechanism=mechanism.name) + + state = GraphStateRoot(repository, root=root) + if not state.path.exists(): + record("context-graph-state", "skip", "no private local graph state exists for this checkout") + else: + try: + state.verify_private() + record("context-graph-state", "pass", "local graph state is private and operator-owned") + except ContextError as error: + record("context-graph-state", "fail", str(error)) + + status = graph_status(state.repository, root=root, revision=revision) + if status.state == "absent": + record("context-graph-generation", "skip", status.detail or "no local graph generation is published") + elif status.usable: + build = {key: value for key, value in status.shareable_summary().get("build", {}).items() + if key != "schema"} + record("context-graph-generation", "pass", + "the published generation binds the current revision", **build) + else: + record("context-graph-generation", "fail", status.detail or f"local graph is {status.state}", + state=status.state) + + ordering = {"fail": 0, "warn": 1, "pass": 2, "skip": 3} + overall = min((check["status"] for check in checks), key=lambda value: ordering[value]) + return { + "schema": "code_mower.contextGraphDoctor.v1", + "status": "fail" if any(check["status"] == "fail" for check in checks) else overall, + "checks": checks, + } + + +def render_status_text(status: GenerationStatus) -> str: + """A short operator-facing summary. Metadata only; no indexed content.""" + lines = [f"Local graph: {status.state}"] + if status.detail: + lines.append(f" {status.detail}") + manifest = status.manifest + if manifest is not None: + lines.extend( + [ + f" generation: {manifest.generation}", + f" commit: {manifest.commit}", + f" tree: {manifest.tree}", + f" provider: {manifest.provider.get('distribution')}=={manifest.provider.get('version')}", + f" built at: {manifest.built_at}", + f" tracked: {manifest.tracked_files} files / {manifest.tracked_bytes} bytes", + f" census: {manifest.census_digest}", + f" graph: {manifest.graph_digest} ({manifest.graph_bytes} bytes)", + f" complete: {manifest.completeness}", + ] + ) + return "\n".join(lines) + "\n" + + +def iter_generations(repository: Path, *, root: Path | None = None) -> Iterator[str]: + """Published and unreferenced generation names, for operator inspection.""" + yield from GraphStateRoot(repository, root=root).generation_names() + + +__all__: Sequence[str] = ( + "ARTIFACT_NAME", + "BuildManifest", + "COMPLETE", + "Containment", + "EXTRACTION_TIMEOUT_SECONDS", + "GenerationStatus", + "GraphStateRoot", + "GraphifyPin", + "IndexRequest", + "IndexResult", + "MANIFEST_SCHEMA", + "MAX_ARTIFACT_BYTES", + "MAX_ARTIFACT_ENTRIES", + "MAX_EXTRACT_OPTIONS", + "MAX_TRACKED_FILES", + "PARTIAL", + "TrackedCensus", + "TrackedEntry", + "build_graph", + "checkout_root", + "containment_mechanism", + "containment_prefix", + "doctor_report", + "git_environment", + "graph_status", + "iter_generations", + "load_manifest", + "load_pin", + "materialize_tracked_files", + "read_tracked_census", + "refuse_lazy_object_fetch", + "remove_graph", + "render_status_text", + "resolve_revision", + "scrubbed_environment", + "subprocess_indexer", + "workspace_id", +) diff --git a/src/code_mower/package_manifest.py b/src/code_mower/package_manifest.py index e3100453..7bb851f0 100644 --- a/src/code_mower/package_manifest.py +++ b/src/code_mower/package_manifest.py @@ -36,6 +36,16 @@ ("src/code_mower/context_readiness.py", "src/code_mower/context_readiness.py", "core"), ("src/code_mower/context_session.py", "src/code_mower/context_session.py", "core"), ("src/code_mower/context_graph.py", "src/code_mower/context_graph.py", "core"), + ( + "src/code_mower/context_graph_lifecycle.py", + "src/code_mower/context_graph_lifecycle.py", + "core", + ), + ( + "src/code_mower/context_graph_command.py", + "src/code_mower/context_graph_command.py", + "core", + ), ("src/code_mower/productivity_report.py", "src/code_mower/productivity_report.py", "core"), ("tools/code_mower_requirements.txt", "requirements/requirements.txt", "tooling"), ("tools/code_mower_calibration.py", "src/code_mower/code_mower_calibration.py", "core"), diff --git a/tests/test_context_graph_lifecycle.py b/tests/test_context_graph_lifecycle.py new file mode 100644 index 00000000..a29a8882 --- /dev/null +++ b/tests/test_context_graph_lifecycle.py @@ -0,0 +1,3186 @@ +"""Offline lifecycle tests for the optional local repository graph (issue #913). + +Every test here builds a real throwaway Git repository and runs the whole +lifecycle against it with an injected indexer. No graph package is installed, +imported, or required, and nothing leaves this machine: the provider seam is a +callable, so the parts this repository is responsible for -- what gets +materialized, what the manifest binds, how a generation is published, and when +a consumer must refuse one -- are all provable locally. + +``NetworkIsolationTests`` is the one place a socket is opened at all. It binds a +listener on loopback in this process and proves a sandboxed child cannot reach +it, which is the only honest way to test a network boundary: an assertion about +proxy variables would have passed on code that had none. +""" + +from __future__ import annotations + +import base64 +import contextlib +import csv +import dataclasses +import hashlib +import io +import json +import os +import socket +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +import unittest +import uuid +from datetime import datetime, timezone +from pathlib import Path +from unittest import mock + +from code_mower import context_graph +from code_mower import context_graph_command as command +from code_mower import context_graph_lifecycle as lifecycle +from code_mower.context_contract import ContextError + + +PIN = lifecycle.GraphifyPin( + distribution="graphifyy", + version="0.9.58", + wheel_sha256="a" * 64, + options=("--no-network",), +) +NOW = datetime(2026, 3, 1, 9, 30, tzinfo=timezone.utc) + + +def git(repository: Path, *arguments: str) -> str: + return subprocess.run( + ["git", "-C", str(repository), *arguments], + check=True, + capture_output=True, + text=True, + env={ + "PATH": os.environ.get("PATH", ""), + "HOME": str(repository), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_NAME": "Test", + "GIT_COMMITTER_EMAIL": "test@example.invalid", + }, + ).stdout + + +def make_repository(root: Path) -> Path: + """A small repository with a tracked file, an ignored file and a secret.""" + repository = root / "checkout" + repository.mkdir() + git(repository, "init", "-q", "-b", "main") + (repository / "example_pkg").mkdir() + (repository / "example_pkg" / "config.py").write_text("VALUE = 1\n", encoding="utf-8") + (repository / "README.md").write_text("# example\n", encoding="utf-8") + (repository / ".gitignore").write_text("scratch/\n", encoding="utf-8") + git(repository, "add", ".") + git(repository, "commit", "-q", "-m", "initial") + # Present in the working tree at build time, and tracked by nothing. + (repository / "scratch").mkdir() + (repository / "scratch" / "notes.txt").write_text("private working note\n", encoding="utf-8") + (repository / "untracked-secret.env").write_text("TOKEN=not-a-real-secret\n", encoding="utf-8") + return repository + + +class FakeChild: + """Enough of ``subprocess.Popen`` to stand in for the provider process. + + The adapter no longer hands the run to ``subprocess.run``: it opens the + child itself so the child can lead its own process group and the whole + group can be stopped if the run overruns. A stand-in is therefore a context + manager that gets waited on, and a ``CompletedProcess`` no longer describes + what the launch produces. + + ``pid`` is deliberately never a live process. A stand-in that carried a real + pid -- this process's own, say -- would have a test signalling a process + group it is itself a member of. + """ + + def __init__(self, returncode: int = 0, *, overruns: bool = False) -> None: + self.returncode = returncode + self.pid = -1 + self.killed = False + self._overruns = overruns + + def __enter__(self) -> "FakeChild": + return self + + def __exit__(self, *exception: object) -> bool: + return False + + def wait(self, timeout: float | None = None) -> int: + if self._overruns: + raise subprocess.TimeoutExpired("graphify", timeout or 0) + return self.returncode + + def poll(self) -> int | None: + # ``None`` means the leader is still running, which is exactly the + # state an overrun leaves it in: the wait gave up on it, not the other + # way round. + return None if self._overruns else self.returncode + + def kill(self) -> None: + self.killed = True + + +def recording_indexer(payload: bytes = b"graph-bytes", *, completeness: str = lifecycle.COMPLETE, + seen: list | None = None, indexed_files: int = 0): + """An indexer that writes a fixed artifact and records what it was shown.""" + + def run(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: + if seen is not None: + seen.append(request) + request.output_path.write_bytes(payload) + return lifecycle.IndexResult(completeness=completeness, indexed_files=indexed_files) + + return run + + +class TemporaryWorkspace(unittest.TestCase): + def setUp(self) -> None: + self._directory = tempfile.TemporaryDirectory() + self.addCleanup(self._directory.cleanup) + self.root = Path(self._directory.name).resolve() + self.state = self.root / "state" + self.repository = make_repository(self.root) + + def build(self, **overrides): + arguments = { + "pin": PIN, + "indexer": recording_indexer(), + "root": self.state, + "now": NOW, + } + arguments.update(overrides) + return lifecycle.build_graph(self.repository, **arguments) + + +class PinTests(unittest.TestCase): + def test_accepts_one_exact_release(self) -> None: + pin = lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": "b" * 64} + ) + self.assertEqual(pin.requirement, "graphifyy==0.9.58") + # A pin that names no options is still a restricted pin: the adoption + # conditions are not the operator's to omit by leaving a field out. + self.assertEqual(pin.options, ("--code-only", "--no-cluster")) + + def test_the_required_extraction_restrictions_are_always_carried(self) -> None: + """Code-only and no-cluster are conditions of the adopt decision. + + They are prepended into the pin rather than added at the launch site, + so the manifest records the run that actually happened. A pin that + already names one keeps exactly one copy of it, and whatever else it + names is preserved after them. + """ + pin = lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": "b" * 64, + "options": ["--no-cluster", "--max-workers", "4"]} + ) + self.assertEqual(pin.options, ("--code-only", "--no-cluster", "--max-workers", "4")) + + def test_rejects_options_that_undo_the_extraction_restrictions(self) -> None: + """An option that re-enables clustering or non-code extraction is refused. + + Overriding it by argument order would leave the pin claiming one thing + and the provider doing another; refusing says so where an operator can + see it. + """ + for option in ("--cluster", "--no-code-only", "--code-only=false", "--no-cluster=0"): + with self.subTest(option=option): + with self.assertRaises(ContextError): + lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": "b" * 64, + "options": [option]} + ) + + def test_the_options_bound_is_applied_to_what_actually_runs(self) -> None: + """The bound counts the normalized tuple, not what the file spelled. + + Counting the raw list let a pin pass validation and then normalize into + one option too many, so its own ``as_metadata`` could no longer be read + back: a build could publish a generation whose manifest is invalid the + moment anything reloads it, pruning the last usable generation for one + nothing can read. The bound now refuses it wherever the pin is made. + """ + source = { + "distribution": "graphifyy", + "version": "0.9.58", + "wheel_sha256": "b" * 64, + "options": [f"--flag-{index}" for index in range(lifecycle.MAX_EXTRACT_OPTIONS - 1)], + } + with self.assertRaises(ContextError): + lifecycle.load_pin(source) + with self.assertRaises(ContextError): + lifecycle.GraphifyPin( + distribution="graphifyy", + version="0.9.58", + wheel_sha256="b" * 64, + options=tuple(source["options"]), + ) + + def test_a_pin_at_the_bound_round_trips_through_its_own_metadata(self) -> None: + """Normalization is a fixed point, so a published manifest can be reread. + + The required flags are dropped wherever they appear and re-prepended + exactly once, so a pin holding the most options it may hold reloads to + an equal pin rather than growing by two each time it is written out. + """ + extra = [f"--flag-{index}" for index in range(lifecycle.MAX_EXTRACT_OPTIONS - 2)] + pin = lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": "b" * 64, + "options": extra} + ) + self.assertEqual(len(pin.options), lifecycle.MAX_EXTRACT_OPTIONS) + self.assertEqual(lifecycle.load_pin(pin.as_metadata()), pin) + self.assertEqual(lifecycle.load_pin(pin.as_metadata()).options, pin.options) + + def test_rejects_ranges_and_unpinned_shapes(self) -> None: + """A range, a marker, or a missing digest lets a build drift silently.""" + for version in (">=0.9", "0.9.*", "latest", "", "0.9.58; python_version>'3'"): + with self.subTest(version=version): + with self.assertRaises(ContextError): + lifecycle.load_pin( + {"distribution": "graphifyy", "version": version, "wheel_sha256": "b" * 64} + ) + + def test_rejects_missing_or_malformed_artifact_digest(self) -> None: + for digest in (None, "", "b" * 63, "not-hex" + "b" * 57): + with self.subTest(digest=digest): + with self.assertRaises(ContextError): + lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": digest} + ) + + def test_rejects_unknown_fields(self) -> None: + with self.assertRaises(ContextError): + lifecycle.load_pin( + {"distribution": "graphifyy", "version": "0.9.58", "wheel_sha256": "b" * 64, + "index_url": "https://example.invalid/simple"} + ) + + +class CensusAndMaterializationTests(TemporaryWorkspace): + def test_census_reads_the_commit_not_the_working_tree(self) -> None: + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertEqual( + [entry.path for entry in census.entries], + [".gitignore", "README.md", "example_pkg/config.py"], + ) + + def test_census_digest_changes_when_tracked_content_changes(self) -> None: + commit, _ = lifecycle.resolve_revision(self.repository) + before = lifecycle.read_tracked_census(self.repository, commit).digest + (self.repository / "README.md").write_text("# example changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + after_commit, _ = lifecycle.resolve_revision(self.repository) + self.assertNotEqual(before, lifecycle.read_tracked_census(self.repository, after_commit).digest) + + def test_symlinks_and_submodules_are_skipped_rather_than_followed(self) -> None: + """A tracked symlink can name a target the build was never shown.""" + os.symlink("/etc/passwd", self.repository / "linked.py") + git(self.repository, "add", "linked.py") + git(self.repository, "commit", "-q", "-m", "symlink") + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertNotIn("linked.py", [entry.path for entry in census.entries]) + self.assertIn(("linked.py", "symlink"), census.skipped) + + def test_committed_provider_state_is_skipped_rather_than_indexed(self) -> None: + """A tracked ``.graphify`` is an old cache, not content to index. + + Materializing it would let the provider resume from a cache built over + content this build never saw, and the adapter would then collect + tracked repository bytes as if the provider had just produced them. + """ + for name in (".graphify", "vendor/.GRAPH"): + directory = self.repository / name + directory.mkdir(parents=True) + (directory / "cache.json").write_text('{"stale": true}\n', encoding="utf-8") + git(self.repository, "add", ".graphify", "vendor") + git(self.repository, "commit", "-q", "-m", "committed provider state") + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertEqual( + [entry.path for entry in census.entries], + [".gitignore", "README.md", "example_pkg/config.py"], + ) + self.assertIn((".graphify/cache.json", "provider state"), census.skipped) + self.assertIn(("vendor/.GRAPH/cache.json", "provider state"), census.skipped) + destination = self.root / "materialized-with-state" + lifecycle.materialize_tracked_files(self.repository, census, destination) + self.assertFalse((destination / ".graphify").exists()) + self.assertFalse((destination / "vendor").exists()) + + def test_committed_code_mower_state_is_skipped_rather_than_indexed(self) -> None: + """A tracked ``.code-mower`` is this tool's own state, not content. + + ``context_graph`` refuses a packet that cites into ``.code-mower`` + because a graph that reached in there escaped the checkout it was asked + to index. The lifecycle has to agree at the other end: if those bytes + are handed to the indexer in the first place, the evidence contract is + refusing a citation to content the provider has already read. + """ + for name in (".code-mower", "vendor/.CODE-MOWER"): + directory = self.repository / name + directory.mkdir(parents=True) + (directory / "packet.json").write_text('{"secret": "packet"}\n', encoding="utf-8") + git(self.repository, "add", ".code-mower", "vendor") + git(self.repository, "commit", "-q", "-m", "committed code mower state") + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertEqual( + [entry.path for entry in census.entries], + [".gitignore", "README.md", "example_pkg/config.py"], + ) + self.assertIn((".code-mower/packet.json", "private state"), census.skipped) + self.assertIn(("vendor/.CODE-MOWER/packet.json", "private state"), census.skipped) + destination = self.root / "materialized-with-code-mower" + lifecycle.materialize_tracked_files(self.repository, census, destination) + self.assertFalse((destination / ".code-mower").exists()) + self.assertFalse((destination / "vendor").exists()) + + def test_the_census_excludes_exactly_the_evidence_contract_roots(self) -> None: + """The two ends of the policy share one set rather than two copies. + + A name added to ``context_graph._EXCLUDED_ROOTS`` must not have to be + remembered here as well, so this asserts identity of the object and not + merely equality of its contents. + """ + self.assertIs(lifecycle._PRIVATE_STATE_ROOTS, context_graph._EXCLUDED_ROOTS) + for root in context_graph._EXCLUDED_ROOTS: + with self.subTest(root=root): + self.assertIsNotNone(lifecycle._private_state_reason(f"vendor/{root}/file.json")) + + def test_committing_provider_state_does_not_move_the_census_digest(self) -> None: + # The manifest binds the digest of what was indexed. Committed provider + # state is not indexed, so it does not enter that digest; it is + # accounted for in ``skipped`` instead. + commit, _ = lifecycle.resolve_revision(self.repository) + before = lifecycle.read_tracked_census(self.repository, commit) + state = self.repository / ".graph" + state.mkdir() + (state / "cache.json").write_text('{"stale": true}\n', encoding="utf-8") + git(self.repository, "add", ".graph") + git(self.repository, "commit", "-q", "-m", "state") + after_commit, _ = lifecycle.resolve_revision(self.repository) + after = lifecycle.read_tracked_census(self.repository, after_commit) + self.assertEqual(before.digest, after.digest) + self.assertNotEqual(before.skipped, after.skipped) + + def test_skipped_paths_are_bounded_like_materialized_ones(self) -> None: + """The file-count budget bounds what is written, not what is recorded. + + A repository of symlinks, submodules, or committed provider state adds + nothing to ``entries`` and so passes the tracked-file budget however + large it gets, while ``skipped`` grows with it. The manifest's + ``skipped_paths`` is validated against the same bound on every read, so + an unbounded census would publish a generation, prune its predecessor, + and then read back ``invalid``. The refusal belongs here, before a build + has done anything. + """ + links = [f"link-{index}.py" for index in range(3)] + for name in links: + os.symlink("/etc/passwd", self.repository / name) + # Named, not ``add .``: the workspace deliberately holds an untracked + # secret, and this test is about the census, not about committing it. + git(self.repository, "add", *links) + git(self.repository, "commit", "-q", "-m", "many symlinks") + commit, _ = lifecycle.resolve_revision(self.repository) + with mock.patch.object(lifecycle, "MAX_SKIPPED_PATHS", 2): + with self.assertRaises(ContextError): + lifecycle.read_tracked_census(self.repository, commit) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertEqual(len(census.skipped), 3) + + def test_materialization_writes_only_tracked_files(self) -> None: + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + destination = self.root / "materialized" + written = lifecycle.materialize_tracked_files(self.repository, census, destination) + present = sorted( + str(path.relative_to(destination)) + for path in destination.rglob("*") + if path.is_file() + ) + self.assertEqual(present, [".gitignore", "README.md", "example_pkg/config.py"]) + self.assertEqual(written, census.total_bytes) + self.assertFalse((destination / "scratch").exists()) + self.assertFalse((destination / "untracked-secret.env").exists()) + self.assertFalse((destination / ".git").exists()) + + def test_materialization_is_private_and_non_executable(self) -> None: + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + destination = self.root / "materialized" + lifecycle.materialize_tracked_files(self.repository, census, destination) + self.assertEqual(stat.S_IMODE(destination.stat().st_mode), 0o700) + for path in destination.rglob("*"): + if path.is_file(): + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o600) + + def test_materialization_refuses_an_existing_directory(self) -> None: + commit, _ = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + destination = self.root / "materialized" + destination.mkdir() + with self.assertRaises(ContextError): + lifecycle.materialize_tracked_files(self.repository, census, destination) + + def test_escaping_census_paths_are_rejected(self) -> None: + escaping = ("/etc/passwd", "../outside.py", "a/../../b.py", ".git/config", + "vendor/.git/config", "a\\b.py", ".graphify/cache.json", + "vendor/.GRAPH/cache.json", ".code-mower/packets/one.json", + "docs/.CODE-MOWER/evidence.json") + for index, path in enumerate(escaping): + with self.subTest(path=path): + census = lifecycle.TrackedCensus( + entries=(lifecycle.TrackedEntry("100644", "0" * 40, path, 1),), + skipped=(), + digest="c" * 64, + ) + with self.assertRaises(ContextError): + lifecycle.materialize_tracked_files( + self.repository, census, self.root / f"escape-{index}" + ) + + +class ScrubbedEnvironmentTests(TemporaryWorkspace): + def test_indexer_never_inherits_ambient_credentials(self) -> None: + seen: list[lifecycle.IndexRequest] = [] + secrets = { + "GITHUB_TOKEN": "not-a-real-token", + "ANTHROPIC_API_KEY": "not-a-real-key", + "AWS_SECRET_ACCESS_KEY": "not-a-real-key", + "GRAPHIFY_API_KEY": "not-a-real-key", + } + previous = {name: os.environ.get(name) for name in secrets} + os.environ.update(secrets) + try: + self.build(indexer=recording_indexer(seen=seen)) + finally: + for name, value in previous.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + environment = seen[0].environment + for name in secrets: + self.assertNotIn(name, environment) + self.assertEqual(environment["no_proxy"], "*") + self.assertEqual(environment["https_proxy"], "") + self.assertEqual(environment["GIT_TERMINAL_PROMPT"], "0") + + def test_provider_home_is_redirected_away_from_the_operator(self) -> None: + seen: list[lifecycle.IndexRequest] = [] + self.build(indexer=recording_indexer(seen=seen)) + home = Path(seen[0].environment["HOME"]) + self.assertNotEqual(home, Path.home()) + self.assertTrue(str(home).startswith(str(self.state))) + + def test_allowlist_drops_everything_it_does_not_name(self) -> None: + environment = lifecycle.scrubbed_environment(home=self.root / "h", temporary=self.root / "t") + allowed = set(lifecycle._ENVIRONMENT_ALLOWLIST) | set(lifecycle._NETWORK_DENY) | { + "HOME", "XDG_CONFIG_HOME", "XDG_CACHE_HOME", "XDG_DATA_HOME", + } + self.assertEqual(set(environment) - allowed, set()) + + +class ListingStreamTests(unittest.TestCase): + """The census consumes its listing as it arrives, bounded, and stops the reader. + + Capturing the whole listing first put a repository's worth of metadata in + this process before any census bound could be checked, so a tree far past + every budget exhausted memory instead of being refused at the budget. The + reader is a stand-in here because a repository large enough to prove the + bound for real would be the thing the bound exists to avoid. + """ + + class Reader: + """Enough of ``Popen`` for the record stream: a pipe and an exit status.""" + + def __init__(self, payload: bytes, returncode: int = 0) -> None: + self.stdout = io.BytesIO(payload) + self.returncode = returncode + self.killed = False + + def wait(self, timeout: float | None = None) -> int: + return self.returncode + + def poll(self) -> int | None: + return self.returncode if self.killed else None + + def kill(self) -> None: + self.killed = True + + def records(self, payload: bytes, returncode: int = 0) -> list[str]: + return list(lifecycle._read_records(self.Reader(payload, returncode))) + + def test_records_are_split_on_the_delimiter(self) -> None: + self.assertEqual(self.records(b"one\0two\0"), ["one", "two"]) + + def test_a_run_of_bytes_past_the_record_bound_is_refused(self) -> None: + # No delimiter, so nothing can be classified and nothing can be + # released: this is exactly the shape that grows without limit. + with self.assertRaises(ContextError): + self.records(b"x" * (lifecycle.MAX_CENSUS_RECORD_BYTES + 1) + b"\0") + + def test_a_stream_that_ends_mid_record_is_refused(self) -> None: + with self.assertRaises(ContextError): + self.records(b"one\0two") + + def test_a_reader_that_failed_is_refused_even_after_a_clean_stream(self) -> None: + with self.assertRaises(ContextError): + self.records(b"one\0", returncode=1) + + def test_a_consumer_that_stops_early_stops_the_reader_with_it(self) -> None: + """A refused census must not leave Git enumerating the rest of the tree.""" + reader = self.Reader(b"one\0two\0") + with mock.patch.object(subprocess, "Popen", lambda *arguments, **keywords: reader): + with self.assertRaises(ContextError): + with lifecycle._git_records(Path("/nonexistent"), "ls-tree") as records: + next(records) + raise ContextError("the consumer reached its budget") + self.assertTrue(reader.stdout.closed) + self.assertTrue(reader.killed) + + +CONNECT_PROBE = """ +import socket +import sys + +try: + socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=5).close() +except OSError: + sys.exit(1) +sys.exit(0) +""" + + +READ_PROBE = """ +import sys + +try: + with open(sys.argv[1], "rb") as handle: + handle.read(1) +except OSError: + sys.exit(1) +sys.exit(0) +""" + +#: Set by the CI job that installs a real isolation mechanism. There, a host +#: without one is a broken job rather than a host that cannot be asked, so the +#: skip becomes a failure: coverage that silently skips is coverage nobody has. +REQUIRE_CONTAINMENT = "CODE_MOWER_REQUIRE_CONTAINMENT" + + +def containment_evidence() -> str: + """Why this host offered no mechanism, in the terms the probe decided in. + + A required job that fails with "none" tells an operator nothing they can + act on: a launcher can be absent, present but untrusted, or present and + trusted and unable to start a child at all. So the failure carries the + control's verdict, each candidate's verdict, and the launcher's own stderr + -- which is where a refused namespace or an impossible bind says so. + """ + lines: list[str] = [] + readable = lifecycle._interpreter_read_paths() + with tempfile.TemporaryDirectory() as scratch: + lines.append(f"control (no prefix): {lifecycle._classify_probe((), cwd=scratch)}") + for name, path in lifecycle._SANDBOX_CANDIDATES: + launcher = lifecycle._trusted_launcher(path) + if launcher is None: + lines.append(f"{name} at {path}: exists={os.path.exists(path)}, not trusted") + continue + prefix = lifecycle._prefix_for( + lifecycle.Containment(name=name, launcher=launcher), + writable=(scratch,), + readable=readable, + ) + started = subprocess.run( + [*prefix, sys.executable, "-c", "print('started')"], + check=False, + capture_output=True, + timeout=120, + cwd=scratch, + ) + lines.append( + f"{name} at {path}: verdict={lifecycle._classify_probe(prefix, cwd=scratch)}" + f" start={started.returncode} stdout={started.stdout[:200]!r}" + f" stderr={started.stderr[:400]!r}" + ) + return "\n".join(lines) + + +def require_containment(test: unittest.TestCase) -> lifecycle.Containment: + mechanism = lifecycle.containment_mechanism() + if mechanism is None: + if os.environ.get(REQUIRE_CONTAINMENT) == "1": + test.fail( + "this job requires a verified isolation mechanism and this host offers none\n" + + containment_evidence() + ) + test.skipTest("this host offers no OS sandbox that contains a child process") + return mechanism + + +def install_provider( + environment: Path, + *, + pin: lifecycle.GraphifyPin = PIN, + script: str = "graphify", + distribution: str | None = None, + version: str | None = None, + body: str = "#!/bin/sh\nexit 0\n", + record: bool = True, + digest: bool = True, +) -> Path: + """A pinned provider the way an installer really leaves one on disk. + + A virtual environment holding a console script and the ``.dist-info`` the + installer wrote for the distribution that provided it: ``METADATA`` naming + the release, and a ``RECORD`` claiming the script by the relative path a + wheel's ``RECORD`` uses for it. The identity check reads exactly this, so a + fixture that faked it would prove nothing. + + ``distribution`` and ``version`` default to the pin's, so the ordinary call + installs what the pin names; naming them differently is how a test installs + something else under the same script name. + """ + name = distribution if distribution is not None else pin.distribution + released = version if version is not None else pin.version + scripts = environment / "bin" + scripts.mkdir(parents=True, exist_ok=True) + (environment / "pyvenv.cfg").write_text("home = /usr/bin\n", encoding="utf-8") + executable = scripts / script + executable.write_text(body, encoding="utf-8") + executable.chmod(0o700) + site_packages = environment / "lib" / "python3.12" / "site-packages" + dist_info = site_packages / f"{name.replace('-', '_')}-{released}.dist-info" + dist_info.mkdir(parents=True, exist_ok=True) + (dist_info / "METADATA").write_text( + f"Metadata-Version: 2.1\nName: {name}\nVersion: {released}\n" + "Summary: a stand-in for a pinned provider\n" + # Past the header block, because the real file carries a description + # and the reader must stop at the blank line rather than keep matching + # lines that look like headers. + "\nVersion: 9.9.9\nName: not-the-pinned-distribution\n", + encoding="utf-8", + ) + if record: + # The relative spelling a wheel uses for a console script: three levels + # up from ``site-packages`` to the environment root, then ``bin``. + recorded = f"../../../bin/{script}" + if digest: + encoded = base64.urlsafe_b64encode( + hashlib.sha256(executable.read_bytes()).digest() + ).decode().rstrip("=") + line = f"{recorded},sha256={encoded},{executable.stat().st_size}\n" + else: + line = f"{recorded},,\n" + (dist_info / "RECORD").write_text( + f"{line}{dist_info.name}/METADATA,,\n{dist_info.name}/RECORD,,\n", encoding="utf-8" + ) + return executable + + +@contextlib.contextmanager +def stand_in_installation(): + """An install whose identity is taken as checked. + + The identity check reads a real ``.dist-info``, which only a host with the + pinned provider installed actually has. Tests about argv, streams, and + collection stand it in for the same reason they stand in the exposure; + ``ProviderIdentityTests`` runs it for real against a real install. + """ + with mock.patch.object(lifecycle, "_verify_provider_installation", lambda command, **_: None): + yield + + +@contextlib.contextmanager +def stand_in_containment(prefix: tuple[str, ...]): + """A verified mechanism whose argv is a fixed stand-in. + + The real prefix is a function of the host, so a test that wants to read the + argv a launch was given -- rather than to prove containment -- pins it. + ``_provider_read_paths`` goes with it: the exposure a build computes names + an install that only a host with the pinned provider on it actually has, + and so does the pin check that install is read for. + """ + with mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ): + with mock.patch.object(lifecycle, "containment_prefix", lambda **keywords: prefix): + with mock.patch.object(lifecycle, "_provider_read_paths", lambda command, **_: ()): + with stand_in_installation(): + yield + + +@contextlib.contextmanager +def no_containment(): + """A host that offers no mechanism at all.""" + with mock.patch.object(lifecycle, "containment_mechanism", lambda: None): + yield + + +class NetworkIsolationTests(unittest.TestCase): + """The provider's network boundary, against a socket that is really there.""" + + def setUp(self) -> None: + self.listener = socket.socket() + self.addCleanup(self.listener.close) + self.listener.bind(("127.0.0.1", 0)) + self.listener.listen(1) + self.port = self.listener.getsockname()[1] + + def connect(self, prefix: tuple[str, ...], *, cwd: Path | None = None) -> int: + # ``cwd`` names a directory inside the exposure, because that is where a + # build's provider starts: the boundary does not expose this process's + # own working directory, and a child asked to start in a directory its + # sandbox does not have never runs at all. + return subprocess.run( + [*prefix, sys.executable, "-c", CONNECT_PROBE, str(self.port)], + check=False, + capture_output=True, + timeout=120, + cwd=None if cwd is None else str(cwd), + ).returncode + + def test_an_unsandboxed_child_reaches_the_listening_socket(self) -> None: + # The control. Without it, a sandboxed child that failed to start for + # some unrelated reason would read as proof of isolation. + self.assertEqual(self.connect(()), 0) + + def real_prefix(self, scratch: Path) -> tuple[str, ...]: + """The argv this host would really confine a build with.""" + require_containment(self) + # A checkout of its own: the readable set is now checked against the + # tree a build would be indexing, so this call has to name one. + return lifecycle.containment_prefix( + writable=(scratch,), + readable=lifecycle._interpreter_read_paths(), + repository=self.root_outside(), + ) + + def test_a_sandboxed_child_cannot_reach_the_listening_socket(self) -> None: + with tempfile.TemporaryDirectory() as scratch: + prefix = self.real_prefix(Path(scratch)) + self.assertEqual(self.connect(prefix, cwd=Path(scratch)), 1) + + def test_the_selected_mechanism_really_contains_a_child_on_this_host(self) -> None: + """The integration control: the real mechanism, not a stand-in for one. + + The synthetic launchers below pin the classification *algorithm* on + every host, including hosts with no sandbox at all. They cannot show + that ``sandbox-exec`` or ``bwrap`` as this module spells them actually + confines anything. Where one of them is available, this runs it for + real: an unconfined child must reach a listener that is really there + and read a secret planted outside its exposure, and a child under the + selected prefix must do neither. + """ + with tempfile.TemporaryDirectory() as scratch: + prefix = self.real_prefix(Path(scratch)) + self.assertEqual(lifecycle._classify_probe((), cwd=scratch), lifecycle._REACHED) + self.assertEqual(lifecycle._classify_probe(prefix, cwd=scratch), lifecycle._CONTAINED) + + def test_the_selected_mechanism_hides_a_file_outside_the_exposure(self) -> None: + """The filesystem half, named separately from the classifier that uses it. + + The reproduction this replaces read an external ignored ``.env`` through + the selected sandbox: the macOS profile denied the network and allowed + the whole host filesystem, and ``--dev-bind / /`` did the same on Linux. + A working directory is not a boundary. What is exposed is exposed; a + secret beside it is not there at all. + """ + with tempfile.TemporaryDirectory() as scratch: + prefix = self.real_prefix(Path(scratch)) + exposed = Path(scratch) / "inside" + exposed.write_text("visible", encoding="utf-8") + hidden = self.root_outside() / ".env" + hidden.write_text("SECRET=planted", encoding="utf-8") + self.assertEqual(self.read_through(prefix, exposed, cwd=Path(scratch)), 0) + self.assertEqual(self.read_through(prefix, hidden, cwd=Path(scratch)), 1) + + def root_outside(self) -> Path: + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + return Path(directory.name) + + def read_through(self, prefix: tuple[str, ...], path: Path, *, cwd: Path | None = None) -> int: + return subprocess.run( + [*prefix, sys.executable, "-c", READ_PROBE, str(path)], + check=False, + capture_output=True, + timeout=120, + cwd=None if cwd is None else str(cwd), + ).returncode + + def test_a_mechanism_that_denies_only_the_network_is_not_a_boundary(self) -> None: + """Half a boundary classifies as none. + + A launcher that gives its child an empty network namespace and leaves + the host filesystem in place is exactly what this module used to select. + The child reports it, and the classification refuses it rather than + recording the half it liked. + """ + launcher = self.network_only_launcher(self.closed_port()) + self.assertEqual(lifecycle._classify_probe((launcher,)), lifecycle._UNUSABLE) + + def network_only_launcher(self, port: int) -> str: + """Redirects the probe at a closed port but leaves the secret readable.""" + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "network-only-launcher" + path.write_text(f'#!/bin/sh\nexec "$1" "$2" "$3" {port} "$5" "$6"\n', encoding="utf-8") + path.chmod(0o700) + return str(path) + + def test_a_launcher_on_PATH_cannot_shadow_a_trusted_one(self) -> None: + """Every candidate is an absolute path, so ``PATH`` decides nothing. + + A launcher resolved through the inherited ``PATH`` can be shadowed by a + program that computes the probe's evidence and reports containment + without establishing any, and the whole verdict is then forged. The + shadow is planted with the names this module looks for and must not be + selected -- nor even consulted. + """ + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + for name in ("bwrap", "unshare", "sandbox-exec"): + shadow = Path(directory.name) / name + shadow.write_text("#!/bin/sh\nexit 40\n", encoding="utf-8") + shadow.chmod(0o700) + for _, candidate in lifecycle._SANDBOX_CANDIDATES: + self.assertTrue(os.path.isabs(candidate), candidate) + self.assertFalse(candidate.startswith(directory.name)) + with mock.patch.dict(os.environ, {"PATH": directory.name}): + with mock.patch.object( + lifecycle, "_classify_probe", lambda prefix, **keywords: lifecycle._REACHED + ): + # Every candidate classifies as "reached", so nothing can be + # selected; the point is that the shadow was never a candidate. + self.assertIsNone(lifecycle._probe_containment()) + + def test_a_launcher_anybody_could_replace_is_not_trusted(self) -> None: + """Ownership and ancestry, not just the file's own mode. + + An executable in a directory somebody else may write can be replaced + between the probe that trusted it and the build that runs it. A + system launcher is the positive control: root-owned, in root-owned + directories nobody else may write. + """ + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + launcher = Path(directory.name) / "launcher" + launcher.write_text("#!/bin/sh\nexit 40\n", encoding="utf-8") + launcher.chmod(0o700) + os.chmod(directory.name, 0o777) + self.assertIsNone(lifecycle._trusted_launcher(str(launcher))) + self.assertIsNone(lifecycle._trusted_launcher("/nonexistent/launcher")) + if not os.path.isfile("/usr/bin/env"): # pragma: no cover - platform + self.skipTest("no system launcher to use as the positive control") + self.assertEqual(lifecycle._trusted_launcher("/usr/bin/env"), "/usr/bin/env") + + def launcher(self, exit_code: int) -> str: + """A stand-in launcher, so the classifier is pinned on every host. + + A host that offers no real mechanism -- a Linux host with unprivileged + user namespaces restricted, say -- would otherwise leave both halves of + the accept/reject decision untested. + """ + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "launcher" + path.write_text(f"#!/bin/sh\nexit {exit_code}\n", encoding="utf-8") + path.chmod(0o700) + return str(path) + + def closed_port(self) -> int: + """A loopback port that refuses connections, as an empty namespace does. + + Bound and never listened on, rather than bound and released: a released + ephemeral port can be handed straight back to the listener this test is + trying to prove unreachable. Holding it means the refusal is the one + this test arranged. + """ + held = socket.socket() + self.addCleanup(held.close) + held.bind(("127.0.0.1", 0)) + return held.getsockname()[1] + + def redirecting_launcher(self, port: int) -> str: + """A stand-in for a launcher that gives its child its own loopback. + + This is the shape ``bwrap --unshare-net`` has: the child really runs, + loopback really comes up inside the new namespace, and the connection is + *refused* because the host's listener is not in there with it. The + launcher runs the probe it was handed against a port nothing is on, + which is what the child would have seen, and points it at a path that + does not exist in place of the planted secret, which is what a child + with no view of the host filesystem sees. ``$6`` is the nonce, + forwarded so the child can still show it ran: a launcher that swallowed + it would be a launcher that did not run the probe. + """ + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "namespace-launcher" + path.write_text( + f'#!/bin/sh\nexec "$1" "$2" "$3" {port} /nonexistent/secret "$6"\n', + encoding="utf-8", + ) + path.chmod(0o700) + return str(path) + + def test_a_launcher_that_never_runs_the_child_is_rejected(self) -> None: + """An exit code is not evidence, and the denial code least of all. + + A launcher that exits with the probe's own "could not connect" code + without executing anything confines nothing, and this is the shape a + broken or hostile launcher has. Nothing reaches the listener either -- + nothing ran -- so a classifier reading the exit code alone would accept + it as a boundary and every build would then run its provider + unconfined. The child's per-run evidence is what separates the two. + """ + self.assertFalse(lifecycle._prefix_confines((self.launcher(lifecycle._PROBE_BASE),))) + + def test_a_child_that_reached_the_network_stack_is_rejected(self) -> None: + self.assertFalse(lifecycle._prefix_confines((self.launcher(lifecycle._PROBE_BASE + lifecycle._PROBE_REACHED_LISTENER),))) + + def test_evidence_from_another_run_does_not_prove_this_one(self) -> None: + """A replayed transcript is not a child that ran. + + The evidence is the digest of a nonce generated for one run, so a + launcher that printed a previous run's evidence -- or one that parroted + its own argv -- says nothing about this run. + """ + stale = hashlib.sha256(b"an-earlier-nonce").hexdigest() + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + path = Path(directory.name) / "replaying-launcher" + path.write_text( + f"#!/bin/sh\necho {stale}\necho \"$@\"\nexit {lifecycle._PROBE_BASE}\n", + encoding="utf-8", + ) + path.chmod(0o700) + self.assertFalse(lifecycle._prefix_confines((str(path),))) + + def test_a_launcher_that_cannot_start_is_rejected(self) -> None: + self.assertFalse(lifecycle._prefix_confines(("/nonexistent/launcher",))) + + def test_a_candidate_that_does_not_deny_the_network_is_rejected(self) -> None: + # ``env`` runs its argument unchanged: a prefix that contains nothing + # must not be mistaken for a boundary just because it launches. + passthrough = ("/usr/bin/env",) + if not os.access(passthrough[0], os.X_OK): # pragma: no cover - platform + self.skipTest("no pass-through launcher to test against") + self.assertFalse(lifecycle._prefix_confines(passthrough)) + + def test_a_refused_connection_inside_a_namespace_is_containment(self) -> None: + """The bubblewrap case: refused by an empty namespace, not by the host. + + Classifying on the child's errno cannot tell that apart from a refusal + by an unused host port, so a probe that only accepted ``EPERM``-shaped + denials rejected working bubblewrap isolation and left such hosts unable + to build at all. The verdict is taken at the listener instead: nothing + arrived, so the child was contained. + """ + self.assertTrue(lifecycle._prefix_confines((self.redirecting_launcher(self.closed_port()),))) + + def test_the_probe_proves_its_own_apparatus_before_trusting_a_refusal(self) -> None: + """No candidate passes if an unsandboxed child cannot reach the listener. + + "Could not connect" only means containment when connecting was possible + in the first place. If the control fails -- no probe interpreter, + loopback unavailable -- every candidate would look like a boundary, so + the whole probe refuses and builds refuse with it. + """ + calls: list[tuple[str, ...]] = [] + + def classify(prefix, **keywords): + # The probe child starts inside the exposure, so every call names a + # working directory; what this test reads is the prefix. + calls.append(tuple(prefix)) + return lifecycle._CONTAINED + + with mock.patch.object(lifecycle, "_classify_probe", classify): + self.assertIsNone(lifecycle._probe_containment()) + # The control ran, and nothing was probed after it failed. + self.assertEqual(calls, [()]) + + +#: What a provider that finished leaves behind: a completion claim and counts +#: that admit nothing outstanding. +FINISHED_REPORT = {"complete": True, "code_files": 3, "requeued": 0} + + +class ProviderLaunchTests(TemporaryWorkspace): + """What ``subprocess_indexer`` actually hands the operating system.""" + + def setUp(self) -> None: + super().setUp() + # Every artifact path this fixture has handed out, newest last, so a + # test that launches more than once can name the run it means. + self.artifacts: list[Path] = [] + # A provider on ``PATH``, because a bare ``--indexer graphify`` is + # resolved once at construction now rather than looked up again by the + # launch. The stand-in sits outside the repository: a provider inside + # the checkout is refused, and that refusal has its own test. + self.provider_directory = Path(tempfile.mkdtemp(dir=self.root)) + self.provider = self.provider_directory / "graphify" + self.provider.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + self.provider.chmod(0o700) + path = mock.patch.dict( + os.environ, + {"PATH": os.pathsep.join([str(self.provider_directory), os.environ.get("PATH", "")])}, + ) + path.start() + self.addCleanup(path.stop) + + def request(self, pin: lifecycle.GraphifyPin = PIN) -> lifecycle.IndexRequest: + # A fresh directory per launch, because that is what a build hands the + # provider: ``materialize_tracked_files`` refuses a destination that + # already exists, and so the provider never sees state from a prior run. + source = Path(tempfile.mkdtemp(dir=self.root)) + # The artifact is fresh for the same reason. A build writes it into the + # generation it is about to publish, and the writer opens it O_EXCL -- + # a path shared between two launches would collide on the second rather + # than exercise anything, so each request names its own. + artifact = Path(tempfile.mkdtemp(dir=self.root)) / "graph.bin" + self.artifacts.append(artifact) + return lifecycle.IndexRequest( + source_root=source, + output_path=artifact, + environment={"PATH": os.environ.get("PATH", "")}, + pin=pin, + commit="a" * 40, + tree="b" * 40, + ) + + def run_indexer( + self, + executable: str, + *, + sandbox=("/sandbox", "--deny"), + report: object = FINISHED_REPORT, + state_directory: str = ".graphify", + pin: lifecycle.GraphifyPin = PIN, + ) -> tuple[list[str], lifecycle.IndexResult]: + """Launch the adapter with the provider's side of the contract faked. + + ``extract`` writes its state beside the sources it was run over, so the + stand-in has to leave that state behind for the adapter to collect -- + an exit status alone is not a finished build. + """ + request = self.request(pin) + recorded: list[list[str]] = [] + # What the adapter asked the operating system for, kept beside the argv + # because the stream arrangement is as much of the contract as it is. + self.launch_options: list[dict[str, object]] = [] + + def fake_popen(argv, **kwargs): + recorded.append(list(argv)) + self.launch_options.append(dict(kwargs)) + if state_directory: + written = request.source_root / state_directory + written.mkdir(exist_ok=True) + (written / "graph.bin").write_bytes(b"graph-bytes") + if report is not None: + (written / "manifest.json").write_text(json.dumps(report), encoding="utf-8") + return FakeChild() + + # The stand-in spans the launch as well as the construction: the + # exposure is built per run, from what *that* run exposes, so the + # prefix is computed inside ``indexer(request)`` and a host with no + # real mechanism would otherwise refuse there. ``Popen`` is patched + # only around the launch, so the Git calls a build makes are never + # intercepted by this stand-in. + with stand_in_containment(sandbox): + indexer = lifecycle.subprocess_indexer(executable, repository=self.repository, pin=pin) + with mock.patch.object(subprocess, "Popen", fake_popen): + result = indexer(request) + return recorded[0], result + + def launched_argv(self, executable: str, *, sandbox=("/sandbox", "--deny")) -> list[str]: + return self.run_indexer(executable, sandbox=sandbox)[0] + + def test_the_provider_is_launched_inside_the_sandbox(self) -> None: + argv = self.launched_argv("graphify") + self.assertEqual(argv[:3], ["/sandbox", "--deny", str(self.provider)]) + + def test_the_exposure_is_built_from_what_this_run_writes(self) -> None: + """The boundary names this request's copy and this request's scratch. + + A prefix computed once, at construction, could not name either: both + are made per build. The launch would then confine the provider to + somewhere other than the tree it was asked to index, and the + redirected ``HOME`` and ``TMPDIR`` the environment points at would be + absent from the child's filesystem view -- a provider that cannot + start. So the exposure is read back from the call itself. + """ + scratch = Path(tempfile.mkdtemp(dir=self.root)) + request = dataclasses.replace(self.request(), writable=(scratch,)) + recorded: list[dict[str, object]] = [] + + def record_prefix(**keywords: object) -> tuple[str, ...]: + recorded.append(dict(keywords)) + return ("/sandbox",) + + def fake_popen(argv, **kwargs): + written = request.source_root / ".graphify" + written.mkdir(exist_ok=True) + (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "manifest.json").write_text(json.dumps(FINISHED_REPORT), encoding="utf-8") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(lifecycle, "containment_prefix", record_prefix): + with mock.patch.object(subprocess, "Popen", fake_popen): + indexer(request) + self.assertEqual(len(recorded), 1) + self.assertEqual(recorded[0]["writable"], (request.source_root, scratch)) + + def test_the_provider_is_given_no_stream_this_process_has_to_hold(self) -> None: + """A talkative indexer must not be able to fill this process's memory. + + Nothing reads the provider's stdout or stderr -- completeness comes + from the report it writes, not from what it printed -- so buffering + them would only accumulate whatever it chose to log, for up to the + timeout, under neither the tracked-content budget nor the artifact + one. Inheriting them instead is not the alternative: diagnostics can + echo indexed source, and this process may be writing JSON to stdout. + """ + self.launched_argv("graphify") + options = self.launch_options[0] + self.assertNotIn("capture_output", options) + self.assertEqual(options["stdout"], subprocess.DEVNULL) + self.assertEqual(options["stderr"], subprocess.DEVNULL) + self.assertEqual(options["stdin"], subprocess.DEVNULL) + + def test_the_provider_leads_its_own_process_group(self) -> None: + """A group, so a timed-out run can be stopped as a whole. + + The signal that ends an overrun has to reach workers the provider + started -- under a launcher such as ``sandbox-exec`` the direct child is + the launcher, not the indexer -- and a group is the only handle on them + this process has. Safe only because no stream is inherited. + """ + self.launched_argv("graphify") + self.assertIs(self.launch_options[0]["start_new_session"], True) + + def test_the_provider_is_invoked_through_its_documented_extract_interface(self) -> None: + # The interface the adopt decision evaluated, recorded in + # docs/graphify-evaluation.md as ``extract`` plus options. An + # ``index --source ... --output ...`` shape would be a different CLI. + argv = self.launched_argv("graphify") + self.assertEqual(argv[3:], ["extract", *PIN.options]) + self.assertNotIn("--source", argv) + self.assertNotIn("--output", argv) + + def test_extraction_is_restricted_to_code_and_never_clusters(self) -> None: + """The adoption conditions are enforced at the launch, not assumed. + + A pin is free to name no options at all, and one that did would + otherwise have launched the provider into clustering and whatever + extraction it does by default -- both of which are separate decisions + nobody has taken. + """ + bare = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) + argv, _ = self.run_indexer("graphify", pin=bare) + self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + + def test_the_launch_restricts_extraction_even_if_the_pin_did_not(self) -> None: + # The pin normalizes its own options, so this reaches past the + # constructor to prove the guarantee does not rest on it alone. + stripped = lifecycle.GraphifyPin(distribution="graphifyy", version="0.9.58", wheel_sha256="a" * 64) + object.__setattr__(stripped, "options", ()) + argv, _ = self.run_indexer("graphify", pin=stripped) + self.assertEqual(argv[3:], ["extract", "--code-only", "--no-cluster"]) + + def test_the_collected_artifact_holds_the_state_the_provider_wrote(self) -> None: + _, result = self.run_indexer("graphify") + self.assertEqual(result.completeness, lifecycle.COMPLETE) + self.assertEqual(result.indexed_files, 3) + names = self.archived_names(self.artifacts[-1].read_bytes()) + self.assertIn("graph.bin", names) + + def archived_names(self, artifact: bytes) -> list[str]: + with tarfile.open(fileobj=io.BytesIO(artifact), mode="r") as archive: + return archive.getnames() + + def test_collecting_the_same_state_twice_produces_the_same_bytes(self) -> None: + # The manifest binds a digest of the artifact, so two builds of one + # commit have to agree on the bytes down to the archive metadata. + self.run_indexer("graphify") + self.run_indexer("graphify") + first, second = (artifact.read_bytes() for artifact in self.artifacts) + self.assertEqual(first, second) + + def test_packing_is_bounded_by_the_archive_and_not_by_the_file_sizes(self) -> None: + """Empty files are not free: their headers are bytes this process holds. + + The budget is enforced on the serialized archive as it is written, so + state whose contents sum to nothing at all is still refused once the + archive it turns into would exceed what this process may allocate. + """ + state = self.root / "packed" + state.mkdir() + for index in range(16): + (state / f"node-{index:02d}.bin").write_bytes(b"") + self.assertEqual(sum(path.stat().st_size for path in state.iterdir()), 0) + with mock.patch.object(lifecycle, "MAX_ARTIFACT_BYTES", 2048): + with self.assertRaises(ContextError): + lifecycle._pack_state(state) + + def test_packing_refuses_more_entries_than_its_budget_allows(self) -> None: + # Bounded while the names are collected, before anything is archived. + state = self.root / "entries" + state.mkdir() + for index in range(4): + (state / f"node-{index}.bin").write_bytes(b"x") + with mock.patch.object(lifecycle, "MAX_ARTIFACT_ENTRIES", 3): + with self.assertRaises(ContextError): + lifecycle._pack_state(state) + self.assertEqual(len(self.archived_names(lifecycle._pack_state(state))), 4) + + def test_a_provider_that_wrote_no_state_publishes_nothing(self) -> None: + with self.assertRaises(ContextError): + self.run_indexer("graphify", state_directory="") + + def test_a_successful_run_with_requeued_entries_is_partial(self) -> None: + # The defect the clean-room run recorded: a repeat that exits zero in + # 1.63 s having requeued 54 entries has not built a complete graph. + _, result = self.run_indexer("graphify", report={"complete": True, "files": 429, "requeued": 54}) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertIn("54 requeued", " ".join(result.notes)) + + def test_a_provider_that_denies_completion_is_partial(self) -> None: + _, result = self.run_indexer("graphify", report={"complete": False, "files": 10}) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_a_run_that_left_no_report_is_partial_rather_than_complete(self) -> None: + # Exit status zero is not completion evidence. Absent evidence resolves + # to the state ``graph_status`` refuses, not the one it accepts. + _, result = self.run_indexer("graphify", report=None) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_an_unparseable_report_is_partial_rather_than_complete(self) -> None: + request = self.request() + + def fake_popen(argv, **kwargs): + written = request.source_root / ".graph" + written.mkdir(exist_ok=True) + (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "manifest.json").write_bytes(b"{not json") + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + result = indexer(request) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_a_report_without_affirmative_completion_evidence_is_partial(self) -> None: + """A document that parses is not a document that claims completion. + + ``{}`` and a report in some schema this adapter does not understand + both say nothing about whether the extraction finished, and nothing is + not a claim. Treating them as complete would hand ``graph_status`` a + usable generation built from an unknown run. + """ + silent = ( + {}, + {"schema": "unexpected"}, + {"code_files": 3}, + {"complete": True}, + {"status": "running", "code_files": 3}, + {"status": "partial", "complete": True, "code_files": 3}, + ) + for report in silent: + with self.subTest(report=report): + _, result = self.run_indexer("graphify", report=report) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + self.assertTrue(result.notes) + + def test_a_report_that_claims_completion_and_counts_its_work_is_complete(self) -> None: + claimed = ( + {"complete": True, "code_files": 3}, + {"status": "success", "indexed_files": 3}, + {"completed": True, "entries": 0, "requeued": 0}, + ) + for report in claimed: + with self.subTest(report=report): + _, result = self.run_indexer("graphify", report=report) + self.assertEqual(result.completeness, lifecycle.COMPLETE) + + def test_an_oversized_report_is_refused_without_being_read_whole(self) -> None: + """Provider output is unbounded input; the read is bounded at the stream. + + Slicing after ``read_bytes()`` would have allocated the whole document + first, so the assertion is not only that the build stays ``partial``: + nothing in the collection path may read a provider file whole. + """ + request = self.request() + oversized = b'{"complete": true, "code_files": 3, "pad": "' + b"x" * lifecycle.MAX_MANIFEST_BYTES + b'"}' + + def fake_popen(argv, **kwargs): + written = request.source_root / ".graphify" + written.mkdir(exist_ok=True) + (written / "graph.bin").write_bytes(b"graph-bytes") + (written / "manifest.json").write_bytes(oversized) + return FakeChild() + + def refuse_whole_file_read(self: Path) -> bytes: + raise AssertionError(f"{self} was read whole") + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with mock.patch.object(Path, "read_bytes", refuse_whole_file_read): + result = indexer(request) + self.assertEqual(result.completeness, lifecycle.PARTIAL) + + def test_extraction_refuses_to_run_over_pre_existing_provider_state(self) -> None: + """State that predates the run is a cache, and would be collected as output. + + The census already keeps committed provider state out of the copy, so + this is the second check rather than the only one -- the source root is + an argument, and everything after the run treats what it finds there as + freshly produced. + """ + request = self.request() + (request.source_root / ".graphify").mkdir() + (request.source_root / ".graphify" / "cache.json").write_text("{}", encoding="utf-8") + launched: list[list[str]] = [] + + def fake_popen(argv, **kwargs): + launched.append(list(argv)) + return FakeChild() + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with self.assertRaises(ContextError): + indexer(request) + self.assertEqual(launched, []) + self.assertFalse(request.output_path.exists()) + + def test_an_overrun_is_stopped_before_it_is_reported_as_one(self) -> None: + """The error arrives after the group is stopped, not before. + + ``build_graph`` deletes the scratch directory as soon as the adapter + raises. If the report came first, a worker that outlived the timeout + would still be writing into a directory being removed underneath it. + """ + request = self.request() + order: list[str] = [] + + def fake_popen(argv, **kwargs): + return FakeChild(overruns=True) + + def record_termination(child, group) -> None: + order.append("stopped") + + with stand_in_containment(("/sandbox",)): + indexer = lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + with mock.patch.object(subprocess, "Popen", fake_popen): + with mock.patch.object(lifecycle, "_terminate_process_group", record_termination): + with self.assertRaises(ContextError) as raised: + indexer(request) + order.append("reported") + self.assertEqual(order, ["stopped", "reported"]) + self.assertIn("time budget", str(raised.exception)) + self.assertFalse(request.output_path.exists()) + + def test_a_host_without_a_sandbox_refuses_to_launch_a_provider(self) -> None: + with no_containment(): + with self.assertRaises(ContextError): + lifecycle.subprocess_indexer("graphify", repository=self.repository, pin=PIN) + + def test_a_relative_provider_path_binds_to_the_invocation_directory(self) -> None: + # The child runs in the materialized copy, so a relative path left + # unresolved would be looked up there instead of where it is installed. + installed = self.root / "venv" / "bin" + installed.mkdir(parents=True) + provider = installed / "graphify" + provider.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + provider.chmod(0o700) + previous = Path.cwd() + os.chdir(self.root) + self.addCleanup(os.chdir, previous) + argv = self.launched_argv(os.path.join("venv", "bin", "graphify")) + self.assertEqual(argv[2], str(provider)) + + def test_a_bare_command_name_is_resolved_once_to_an_absolute_path(self) -> None: + """One lookup, kept, rather than a name the launch looks up again.""" + self.assertEqual(lifecycle._resolved_executable("graphify"), str(self.provider)) + + def test_a_bare_name_on_a_relative_PATH_entry_binds_to_the_invocation_directory(self) -> None: + """The finding: ``PATH=provider-venv/bin`` names two directories. + + A relative ``PATH`` entry is resolved against whoever is looking, and + the launch looks from the materialized copy rather than from where the + operator invoked. Containment was drawn around the install this process + found while the child searched somewhere else, so a correctly installed + provider never launched -- and a repository carrying that same relative + path would have answered the child's search with a tracked file, which + is a build executing content it was only meant to read. + + Both hazards are arranged at once: the provider is installed under the + invocation directory, and a *different* executable of the same name is + planted at the same relative path inside the materialized copy. + """ + installed = self.root / "invoked-from" / "provider-venv" / "bin" + installed.mkdir(parents=True) + provider = installed / "graphify" + provider.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + provider.chmod(0o700) + previous = Path.cwd() + os.chdir(self.root / "invoked-from") + self.addCleanup(os.chdir, previous) + request = self.request() + decoy = request.source_root / "provider-venv" / "bin" + decoy.mkdir(parents=True) + (decoy / "graphify").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") + (decoy / "graphify").chmod(0o700) + with mock.patch.dict(os.environ, {"PATH": os.path.join("provider-venv", "bin")}): + resolved = lifecycle._resolved_executable("graphify") + self.assertEqual(resolved, str(provider)) + self.assertNotEqual(resolved, str(decoy / "graphify")) + + def test_a_bare_name_that_is_on_no_PATH_entry_is_refused(self) -> None: + with mock.patch.dict(os.environ, {"PATH": str(self.root / "empty")}): + with self.assertRaises(ContextError): + lifecycle._resolved_executable("graphify") + + def test_an_unnamed_provider_is_refused(self) -> None: + with self.assertRaises(ContextError): + lifecycle._resolved_executable("") + + +class ProviderExposureFixture(TemporaryWorkspace): + """Shared scaffolding for the two halves of the provider exposure rule.""" + + def script(self, path: Path, *, venv: bool = False, config: str | None = None) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + if venv or config is not None: + (path.parent.parent / "pyvenv.cfg").write_text( + "home = /usr/bin\n" if config is None else config, encoding="utf-8" + ) + path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + path.chmod(0o700) + return path + + def interpreter(self, prefix: Path) -> Path: + """A base installation the way a separately installed Python really looks.""" + (prefix / "bin").mkdir(parents=True, exist_ok=True) + (prefix / "lib").mkdir(parents=True, exist_ok=True) + (prefix / "bin" / "python3").write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + return prefix + + def exposure(self, path: Path) -> tuple[str, ...]: + return lifecycle._provider_read_paths(str(path), repository=self.repository) + + +class ProviderExposureTests(ProviderExposureFixture): + """What the sandbox is told to make readable for the provider itself. + + The exposure used to be the executable's parent and grandparent, which is a + guess about a layout rather than knowledge of one. These hold the rule to + the layouts where that guess was widest: a script in a home directory, a + script directly under a top-level prefix, and a provider living inside the + checkout being indexed. + """ + + def test_a_pinned_virtual_environment_exposes_that_environment_and_no_more(self) -> None: + environment = self.root / "venv" + provider = self.script(environment / "bin" / "graphify", venv=True) + exposed = self.exposure(provider) + self.assertIn(str(environment), exposed) + self.assertIn(str(provider), exposed) + # Neither the directory the environment sits in nor anything above it. + self.assertNotIn(str(self.root), exposed) + self.assertNotIn(str(environment.parent), exposed) + + def test_a_script_in_a_home_directory_does_not_expose_the_home_directory(self) -> None: + """``~/bin/graphify``: the grandparent is the operator's whole home.""" + home = self.root / "home" + provider = self.script(home / "bin" / "graphify") + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_script_directly_under_a_top_level_prefix_does_not_expose_the_root(self) -> None: + """``/opt/graphify``: the grandparent the old heuristic exposed is ``/``. + + Refused on the layout, before the question of how wide the grandparent + happens to be arises: a directory that is not a script directory beside + a ``pyvenv.cfg`` is not an install this module can draw a boundary + around, whatever it contains. + """ + provider = self.script(self.root / "opt" / "graphify") + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_provider_inside_the_checkout_does_not_expose_the_working_tree(self) -> None: + """A proven layout is still refused when the layout is the live checkout. + + This one carries a real ``pyvenv.cfg``, so the layout proof passes and + the refusal has to come from where the environment *is*: inside the + repository, whose ignored files are the thing the materialized copy + exists to keep away from the provider. + """ + provider = self.script(self.repository / ".venv" / "bin" / "graphify", venv=True) + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_provider_inside_the_checkout_is_refused_from_a_subdirectory(self) -> None: + """The refusal is about the checkout, not about the invocation directory. + + ``subprocess_indexer`` is handed whatever ``--repo-path`` was, and the + default is the current directory. Drawing this boundary around + ``/src`` would leave ``/provider-venv`` outside it + -- the live working tree exposed to the provider for no reason other + than which directory the operator happened to be standing in. + """ + provider = self.script(self.repository / "provider-venv" / "bin" / "graphify", venv=True) + outside = self.script(self.root / "elsewhere-venv" / "bin" / "graphify", venv=True) + # The identity check is stood in for: what is under test is where the + # install *is*, and neither of these stand-ins was installed by + # anything. ``ProviderIdentityTests`` runs that check for real. + with stand_in_installation(), mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ): + with self.assertRaises(ContextError): + lifecycle.subprocess_indexer( + str(provider), repository=self.repository / "example_pkg", pin=PIN + ) + # The positive control: an install outside the checkout is still + # accepted from the same subdirectory, so the refusal above is + # about where the provider lives and not about normalizing at all. + self.assertTrue( + callable( + lifecycle.subprocess_indexer( + str(outside), repository=self.repository / "example_pkg", pin=PIN + ) + ) + ) + + def test_an_environment_that_is_the_home_directory_is_refused(self) -> None: + home = self.root / "home" + provider = self.script(home / "bin" / "graphify", venv=True) + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_provider_in_the_system_runtime_asks_for_no_extra_exposure(self) -> None: + """Already inside the read-only runtime every child gets. + + Exposing the enclosing prefix would widen that exposure rather than + narrow it, so nothing but the executable's own spellings is added. + """ + exposed = lifecycle._provider_read_paths("/bin/sh", repository=self.repository) + self.assertEqual(set(exposed), {"/bin/sh", os.path.realpath("/bin/sh")}) + + def test_a_provider_that_cannot_be_located_is_refused_rather_than_guessed_at(self) -> None: + with self.assertRaises(ContextError): + lifecycle._provider_read_paths( + "code-mower-no-such-provider", repository=self.repository + ) + + +class ProviderBaseRuntimeTests(ProviderExposureFixture): + """Whose interpreter gets exposed when the provider was pinned with another. + + The exposure used to name ``sys.base_prefix`` -- the interpreter running + Code Mower. That is the provider's base runtime only by coincidence, on a + host where both were installed by the same thing. Pin a provider with a + ``uv``-managed or otherwise separately installed Python and the child got a + runtime it does not use exposed and its own left out of its filesystem view + entirely, which is a correctly pinned install failing every build inside + the loader. + """ + + def test_the_providers_own_base_interpreter_is_exposed_not_this_processs(self) -> None: + """The finding, reproduced: a ``uv``-managed base outside every system path.""" + managed = self.interpreter(self.root / "uv" / "python" / "cpython-3.13.1") + environment = self.root / "venv" + provider = self.script( + environment / "bin" / "graphify", + config=( + f"home = {managed}/bin\n" + "implementation = CPython\n" + f"base-prefix = {managed}\n" + f"base-exec-prefix = {managed}\n" + f"base-executable = {managed}/bin/python3\n" + ), + ) + exposed = self.exposure(provider) + self.assertIn(str(managed), exposed) + # And not the runtime this process happens to be running under, which + # is what the child was being handed instead of its own. + outsider = self.interpreter(self.root / "elsewhere" / "python") + with mock.patch.object(sys, "base_prefix", str(outsider)): + self.assertNotIn(str(outsider), self.exposure(provider)) + + def test_a_standard_library_environment_names_its_base_through_home_alone(self) -> None: + """``venv`` before 3.11 records ``home`` and nothing else about the base.""" + managed = self.interpreter(self.root / "pythons" / "3.12") + provider = self.script( + self.root / "venv" / "bin" / "graphify", + config=f"home = {managed}/bin\ninclude-system-site-packages = false\n", + ) + self.assertIn(str(managed), self.exposure(provider)) + + def test_a_base_interpreter_in_the_system_runtime_adds_no_exposure(self) -> None: + """Already readable for every child; naming it again would only widen.""" + environment = self.root / "venv" + provider = self.script(environment / "bin" / "graphify", config="home = /usr/bin\n") + self.assertEqual(set(self.exposure(provider)), {str(provider), str(environment)}) + + def test_a_base_prefix_that_is_the_home_directory_is_refused(self) -> None: + """A record read out of a file is not narrower than the same guess.""" + home = self.interpreter(self.root / "home") + provider = self.script( + self.root / "venv" / "bin" / "graphify", config=f"base-prefix = {home}\n" + ) + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_base_prefix_inside_the_checkout_is_refused(self) -> None: + base = self.interpreter(self.repository / "vendor" / "python") + provider = self.script( + self.root / "venv" / "bin" / "graphify", config=f"base-prefix = {base}\n" + ) + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_stale_record_does_not_sink_a_live_one(self) -> None: + """A base interpreter moved since creation, with another key still good.""" + managed = self.interpreter(self.root / "pythons" / "3.13") + provider = self.script( + self.root / "venv" / "bin" / "graphify", + config=( + f"base-prefix = {self.root}/pythons/removed\n" + f"base-executable = {managed}/bin/python3\n" + ), + ) + exposed = self.exposure(provider) + self.assertIn(str(managed), exposed) + self.assertNotIn(f"{self.root}/pythons/removed", exposed) + + def test_an_environment_recording_no_usable_base_is_refused(self) -> None: + """Refused with an instruction rather than built against a guessed runtime. + + Every spelling of the base is absent or stale, so there is nothing to + expose and no reason to believe a build would work. Failing here names + the environment; failing later is a ``SIGABRT`` out of the loader. + """ + provider = self.script( + self.root / "venv" / "bin" / "graphify", + config=f"implementation = CPython\nhome = {self.root}/gone/bin\n", + ) + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_relative_or_unparseable_record_is_not_treated_as_a_path(self) -> None: + provider = self.script( + self.root / "venv" / "bin" / "graphify", + config="base-prefix = ../../etc\nthis line has no separator\n", + ) + with self.assertRaises(ContextError): + self.exposure(provider) + + def test_a_base_prefix_that_contains_the_runtime_exposes_no_more_than_the_runtime(self) -> None: + """``/usr``: the ordinary base for an environment made from system Python. + + Exposing it whole is what the unconditional runtime list used to do by + another route -- and it carries ``/usr/local``, ``/usr/src``, and + anything else the host keeps there. A prefix that *contains* the + read-only runtime is narrowed to the runtime directories inside it, and + on a prefix like that one those are already what every child gets, so + the exposure does not grow at all. + """ + base = self.interpreter(self.root / "system") + (base / "local" / "src" / "someones-checkout").mkdir(parents=True) + provider = self.script( + self.root / "venv" / "bin" / "graphify", config=f"base-prefix = {base}\n" + ) + runtime = (str(base / "lib"), str(base / "bin")) + with mock.patch.object(lifecycle, "_SYSTEM_READ_PATHS", runtime): + exposed = self.exposure(provider) + self.assertNotIn(str(base), exposed) + self.assertNotIn(str(base / "local"), exposed) + # And nothing new: both runtime directories are already exposed to every + # child, so the narrowing adds no path rather than a narrower one. + self.assertEqual( + [path for path in exposed if str(base) in path], + [], + ) + + def test_a_runtime_directory_the_prefix_holds_but_the_runtime_does_not_is_exposed(self) -> None: + """The narrowing exposes what is missing, rather than nothing at all. + + A prefix wide enough to contain one runtime path may still hold the + interpreter's own library directory somewhere the runtime list does not + name, and a child that cannot read it does not start. + """ + base = self.interpreter(self.root / "system") + provider = self.script( + self.root / "venv" / "bin" / "graphify", config=f"base-prefix = {base}\n" + ) + with mock.patch.object(lifecycle, "_SYSTEM_READ_PATHS", (str(base / "bin"),)): + exposed = self.exposure(provider) + self.assertIn(str(base / "lib"), exposed) + self.assertNotIn(str(base), exposed) + + def test_an_implausibly_large_config_is_refused_rather_than_read_whole(self) -> None: + environment = self.root / "venv" + provider = self.script(environment / "bin" / "graphify", venv=True) + (environment / "pyvenv.cfg").write_text( + "# " + "x" * (lifecycle._MAX_VENV_CONFIG_BYTES + 1) + "\n", encoding="utf-8" + ) + with self.assertRaises(ContextError): + self.exposure(provider) + + +class RuntimeExposureTests(ProviderExposureFixture): + """The read-only runtime every build adds, and who checks it. + + This list used to be ``/usr``, ``/etc`` and ``/Library``, added to every + exposure unconditionally, which meant the refusals that guard an exposure + root never saw the set the child was actually confined to: a checkout under + ``/usr/local/src`` stayed readable beside the materialized copy that exists + to replace it, and every machine-wide credential under ``/Library`` and + ``/etc`` with it. + """ + + #: Directories that carry operator data, checkouts, or machine-wide + #: credentials rather than a runtime. Naming any of them in the runtime list + #: is the finding, whatever else changes around it. + FORBIDDEN = ( + "/", + "/usr", + "/usr/local", + "/etc", + "/private/etc", + "/Library", + "/Library/Keychains", + "/Library/Preferences", + "/Library/Application Support", + "/home", + "/Users", + "/opt", + "/var", + "/private/var", + "/tmp", + "/root", + "/mnt", + "/media", + "/srv", + ) + + def test_the_runtime_list_names_runtime_directories_and_not_whole_worlds(self) -> None: + named = set(lifecycle._SYSTEM_READ_PATHS) + for forbidden in self.FORBIDDEN: + self.assertNotIn(forbidden, named) + for path in lifecycle._SYSTEM_READ_PATHS: + # Absolute and already normalized: a runtime path is a constant in + # this module, not something resolved against a caller's directory. + self.assertTrue(os.path.isabs(path), path) + self.assertEqual(os.path.normpath(path), path) + + def test_a_runtime_path_that_contains_the_checkout_refuses_the_build(self) -> None: + """The finding, reproduced against the set rather than against one root. + + Nothing the *caller* asks for is wrong here -- the readable set it + passes is empty. The runtime list alone puts the live working tree + inside the child's filesystem view, which is the case no per-root check + could ever see. + """ + with mock.patch.object(lifecycle, "_SYSTEM_READ_PATHS", (str(self.root),)): + with mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ): + with self.assertRaises(ContextError) as raised: + lifecycle.containment_prefix( + writable=(self.root / "scratch",), + readable=(), + repository=self.repository, + ) + self.assertIn("checkout", str(raised.exception)) + + def test_the_real_runtime_list_is_accepted_on_this_host(self) -> None: + """The control, and a guard on the list itself. + + A runtime path that is the operator's home, an ancestor of it, or an + ancestor of an ordinary checkout would refuse every build on this host + rather than expose anything, so the check needs a positive case on the + real list: the refusal above must come from the planted path and not + from something this module ships. + """ + with mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ): + with mock.patch.object(lifecycle, "_prefix_for", lambda *_, **__: ("/sandbox",)): + with mock.patch.object(lifecycle, "_prefix_confines", lambda *_, **__: True): + prefix = lifecycle.containment_prefix( + writable=(self.root,), readable=(), repository=self.repository + ) + self.assertEqual(prefix, ("/sandbox",)) + + def test_a_readable_set_reaching_the_home_directory_is_refused(self) -> None: + home = self.root / "home" + home.mkdir() + with mock.patch.object(Path, "home", staticmethod(lambda: home)): + with mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ): + with self.assertRaises(ContextError): + lifecycle.containment_prefix( + writable=(self.root,), readable=(home,), repository=self.repository + ) + + +class ProviderIdentityTests(TemporaryWorkspace): + """The pin is checked against the install, before the install is run. + + The manifest records the pin as the provenance of every byte in a + generation. Until this check existed that record was a copy of what the + operator typed: ``--indexer`` named an install, ``--pin-file`` named a + release, and nothing compared them, so a build could publish a manifest + naming graphifyy 0.9.58 over a graph some other release -- or some other + distribution answering to ``extract`` -- had produced. + + Every test here runs the real check against a real ``.dist-info``. Nothing + is stood in but the sandbox, which is a property of the host rather than of + the install. + """ + + def setUp(self) -> None: + super().setUp() + self.sandbox = mock.patch.object( + lifecycle, "containment_mechanism", lambda: lifecycle.Containment("stand-in", "/sandbox") + ) + self.sandbox.start() + self.addCleanup(self.sandbox.stop) + + def adapter(self, executable: Path, *, pin: lifecycle.GraphifyPin = PIN): + return lifecycle.subprocess_indexer(str(executable), repository=self.repository, pin=pin) + + def test_the_pinned_release_installed_where_it_is_named_is_accepted(self) -> None: + """The control. Without it every refusal below could be the fixture.""" + provider = install_provider(self.root / "venv") + self.assertTrue(callable(self.adapter(provider))) + + def test_a_different_version_of_the_pinned_distribution_is_refused(self) -> None: + """The manifest would otherwise name 0.9.58 over a 0.9.57 graph.""" + provider = install_provider(self.root / "venv", version="0.9.57") + with self.assertRaises(ContextError) as raised: + self.adapter(provider) + self.assertIn("0.9.57", str(raised.exception)) + + def test_a_different_distribution_under_the_same_script_name_is_refused(self) -> None: + """The substitution the pin exists to make identifiable. + + ``graphify`` and ``graphifyy`` differ by one character and both may + ship a console script called ``graphify``. Checking the file's name -- + or that *something* by that name is installed -- would accept either. + """ + provider = install_provider(self.root / "venv", distribution="graphify") + with self.assertRaises(ContextError) as raised: + self.adapter(provider) + # Both names spelled out, because one is a prefix of the other: a + # message naming only "graphify" would read as a match. + self.assertIn("is graphify 0.9.58, not the pinned graphifyy 0.9.58", str(raised.exception)) + + def test_an_executable_installed_by_another_distribution_in_the_same_environment_is_refused( + self, + ) -> None: + """The pinned release *is* installed here; the executable is not its own. + + This is the shape the finding named: the environment satisfies the pin, + so any check that asked only "is the pinned release installed?" passes, + while ``--indexer`` names a console script a different distribution + wrote. Ownership is read from ``RECORD``, so the answer is about this + file rather than about the environment around it. + """ + environment = self.root / "venv" + install_provider(environment) + other = install_provider( + environment, script="graphify-lookalike", distribution="something-else", version="1.0" + ) + with self.assertRaises(ContextError): + self.adapter(other) + # The positive control, in the same environment: the pinned release's + # own script is still accepted, so the refusal is about which + # distribution installed the file and not about there being two. + self.assertTrue(callable(self.adapter(environment / "bin" / "graphify"))) + + def test_an_executable_no_installed_distribution_claims_is_refused(self) -> None: + """A script dropped into a pinned environment's ``bin`` is not the pin. + + The environment is otherwise exactly right -- ``pyvenv.cfg``, the + pinned ``.dist-info``, the lot -- and the executable named is simply + not part of it. + """ + environment = self.root / "venv" + install_provider(environment) + loose = environment / "bin" / "handmade" + loose.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + loose.chmod(0o700) + with self.assertRaises(ContextError): + self.adapter(loose) + + def test_an_install_that_records_no_name_and_version_is_refused(self) -> None: + """Unreadable identity fails closed rather than passing unchecked.""" + provider = install_provider(self.root / "venv") + metadata = next((self.root / "venv").glob("lib/*/site-packages/*.dist-info/METADATA")) + metadata.write_text("Metadata-Version: 2.1\n", encoding="utf-8") + with self.assertRaises(ContextError): + self.adapter(provider) + + def test_the_name_is_compared_the_way_an_installer_normalizes_it(self) -> None: + """``Graphify_Y`` and ``graphify-y`` are one distribution (PEP 503). + + A pin must not fail against its own install over the spelling whoever + packaged it happened to use. + """ + pin = dataclasses.replace(PIN, distribution="graphify-y") + provider = install_provider(self.root / "venv", pin=pin, distribution="Graphify_Y") + self.assertTrue(callable(self.adapter(provider, pin=pin))) + + def test_an_executable_modified_since_it_was_installed_is_refused(self) -> None: + """The pinned release is installed and this file is no longer it. + + An install whose console script was rewritten in place still carries + the ``.dist-info`` of the release it was, so name and version alone + would accept it. The installer recorded what it wrote; that is what the + file is held to. + """ + provider = install_provider(self.root / "venv") + provider.write_text("#!/bin/sh\nexit 7\n", encoding="utf-8") + with self.assertRaises(ContextError) as raised: + self.adapter(provider) + self.assertIn("modified", str(raised.exception)) + + def test_an_install_whose_record_carries_no_digest_still_verifies(self) -> None: + """A missing hash is not a mismatch. + + ``RECORD`` is allowed to carry an empty digest -- it cannot record its + own -- and some installers write entries that way. Refusing there would + refuse a correctly pinned install for something that is not evidence of + anything, so ownership and version stand on their own. + """ + provider = install_provider(self.root / "venv", digest=False) + self.assertTrue(callable(self.adapter(provider))) + + def test_the_check_reads_the_install_rather_than_running_the_provider(self) -> None: + """Asking ``graphify --version`` would run the file in question. + + It would also run it outside the sandbox that exists to confine it, and + take its own word for who it is. Nothing may be launched before the + adapter is built. + """ + provider = install_provider(self.root / "venv") + + def refuse_launch(argv, **kwargs): + raise AssertionError(f"the identity check launched {argv}") + + # The check itself, not the whole adapter: building one resolves the + # checkout root, and that is a Git call this assertion would otherwise + # be mistaken for a provider launch. + with mock.patch.object(subprocess, "Popen", refuse_launch): + lifecycle._verify_provider_installation(str(provider), pin=PIN) + + def test_a_build_refuses_a_pin_the_adapter_was_not_checked_against(self) -> None: + """One caller passes both, so a divergence is a miswiring. + + The launch reads the request's options and the manifest records the + request's pin, so an adapter checked against one pin and driven with + another would publish provenance nothing verified -- the same defect + one seam over. + """ + provider = install_provider(self.root / "venv") + indexer = self.adapter(provider) + request = lifecycle.IndexRequest( + source_root=self.root / "absent", + output_path=self.root / "absent" / "graph.bin", + environment={}, + pin=dataclasses.replace(PIN, version="0.9.57"), + commit="a" * 40, + tree="b" * 40, + ) + with self.assertRaises(ContextError): + indexer(request) + + def test_an_unparseable_record_claims_nothing_and_the_build_is_refused(self) -> None: + """A ``RECORD`` that cannot be read is not a ``RECORD`` that vouches. + + Nothing then owns the executable, so the refusal is the same one an + uninstalled script gets: an unreadable install fails closed rather than + passing unchecked. + """ + provider = install_provider(self.root / "venv") + dist_info = next((self.root / "venv").glob("lib/*/site-packages/*.dist-info")) + + def unparseable(stream, *arguments, **keywords): + raise csv.Error("line contains NUL") + + with mock.patch.object(csv, "reader", unparseable): + self.assertEqual(lifecycle._record_entries(dist_info), ()) + with self.assertRaises(ContextError): + self.adapter(provider) + + def test_an_oversized_record_is_refused_without_being_read_whole(self) -> None: + """``RECORD`` is a foreign file, and is bounded at the stream like one.""" + provider = install_provider(self.root / "venv") + record = next((self.root / "venv").glob("lib/*/site-packages/*.dist-info/RECORD")) + record.write_text("x" * (lifecycle._MAX_DIST_RECORD_BYTES + 1), encoding="utf-8") + + def refuse_whole_file_read(self: Path) -> bytes: + raise AssertionError(f"{self} was read whole") + + with mock.patch.object(Path, "read_bytes", refuse_whole_file_read): + with self.assertRaises(ContextError): + self.adapter(provider) + + +@unittest.skipUnless(hasattr(os, "killpg"), "process groups are a POSIX facility") +class ExtractionOverrunTests(unittest.TestCase): + """What a timed-out provider leaves running, proved against real processes. + + An assertion about which signal was sent would have passed on code that + signalled only the direct child, which is the whole defect: an indexer that + forks workers -- and a launcher such as ``sandbox-exec``, where the direct + child is the launcher rather than the indexer -- outlives that signal and + keeps writing into a scratch directory the build is about to delete. + """ + + #: Stands in for a provider that starts a worker and then overruns. The + #: worker's pid is written where the test can read it, because the point is + #: what happens to a process this module never had a handle on. + PROVIDER = """ +import subprocess +import sys +import time + +worker = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) +with open(sys.argv[1], "w") as handle: + handle.write(str(worker.pid)) +time.sleep(300) +""" + + #: Stands in for a provider that starts a worker and then exits on its own, + #: leaving the worker running. The leader's exit is ordinary -- it even + #: reports a status -- and says nothing about whether the work has stopped. + ABANDONS_WORKER = """ +import subprocess +import sys + +worker = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(300)"]) +with open(sys.argv[1], "w") as handle: + handle.write(str(worker.pid)) +raise SystemExit(5) +""" + + def reaped(self, pid: int, *, within: float = 15.0) -> bool: + deadline = time.monotonic() + within + while time.monotonic() < deadline: + try: + os.kill(pid, 0) + except OSError: + return True + time.sleep(0.05) + return False + + def test_a_group_that_cannot_be_established_as_empty_fails_the_run(self) -> None: + """Sending SIGKILL is not the group being gone. + + The caller's next act is to pack or delete the state these processes + are writing, so returning on the strength of a signal that was sent -- + rather than on a group that was observed empty -- hands the rest of the + build a race it cannot see. A group that outlasts the kill fails the + run instead. + """ + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object(lifecycle, "_group_is_empty", lambda group: False): + with mock.patch.object(lifecycle, "_await_group_exit", lambda group, timeout: None): + with self.assertRaises(ContextError) as raised: + lifecycle._run_contained( + [sys.executable, "-c", "pass"], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=30.0, + ) + self.assertIn("could not be stopped", str(raised.exception)) + + def test_a_timed_out_run_takes_the_workers_it_started_with_it(self) -> None: + with tempfile.TemporaryDirectory() as directory: + recorded = Path(directory) / "worker.pid" + with self.assertRaises(subprocess.TimeoutExpired): + lifecycle._run_contained( + [sys.executable, "-c", self.PROVIDER, str(recorded)], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=3.0, + ) + worker = int(recorded.read_text(encoding="utf-8")) + self.assertNotEqual(worker, os.getpid()) + self.assertTrue(self.reaped(worker), "a worker outlived the run that started it") + + def test_a_cancelled_run_takes_the_workers_it_started_with_it(self) -> None: + """Ctrl-C is not the timeout, and the provider never sees the signal. + + The child leads its own session, so the terminal's SIGINT reaches this + process and not the provider. Cleaning up only on ``TimeoutExpired`` + left ``KeyboardInterrupt`` to unwind past a running provider and its + workers, while ``build_graph`` deleted the scratch directory they were + writing into. Every exit from the wait stops the group now, not just + the one the timeout takes. + """ + original = subprocess.Popen.wait + cancelled: list[bool] = [] + + def wait(child, timeout=None): + if cancelled: + return original(child, timeout=timeout) + cancelled.append(True) + # Interrupt once the provider has a worker to abandon; an interrupt + # delivered before that would prove nothing about descendants. + deadline = time.monotonic() + 30.0 + while time.monotonic() < deadline: + try: + if recorded.read_text(encoding="utf-8").strip(): + break + except OSError: + pass + time.sleep(0.05) + raise KeyboardInterrupt + + with tempfile.TemporaryDirectory() as directory: + recorded = Path(directory) / "worker.pid" + with mock.patch.object(subprocess.Popen, "wait", wait): + with self.assertRaises(KeyboardInterrupt): + lifecycle._run_contained( + [sys.executable, "-c", self.PROVIDER, str(recorded)], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=300.0, + ) + worker = int(recorded.read_text(encoding="utf-8")) + self.assertNotEqual(worker, os.getpid()) + self.assertTrue(self.reaped(worker), "a worker outlived the run that was cancelled") + + def test_a_provider_that_exits_leaves_no_worker_behind_it(self) -> None: + """A normal exit is not evidence that the work stopped. + + Cleaning up only on the timeout and the interrupt left the ordinary + ending -- the leader returning a status while a worker it started is + still running -- to be trusted. ``subprocess_indexer`` would then pack + an artifact being concurrently modified, and ``build_graph`` would + delete a scratch directory still being written into. The status still + comes back; it now means what it appears to mean. + """ + with tempfile.TemporaryDirectory() as directory: + recorded = Path(directory) / "worker.pid" + returncode = lifecycle._run_contained( + [sys.executable, "-c", self.ABANDONS_WORKER, str(recorded)], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=60.0, + ) + self.assertEqual(returncode, 5) + worker = int(recorded.read_text(encoding="utf-8")) + self.assertNotEqual(worker, os.getpid()) + self.assertTrue( + self.reaped(worker), + "a worker outlived the provider that started it and returned", + ) + + def test_a_run_that_finishes_in_time_reports_its_own_status(self) -> None: + # The containment is not a behaviour change for an ordinary run: the + # exit status still comes back, and nothing is signalled. + with tempfile.TemporaryDirectory() as directory: + returncode = lifecycle._run_contained( + [sys.executable, "-c", "raise SystemExit(3)"], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=60.0, + ) + self.assertEqual(returncode, 3) + + def test_a_run_that_leaves_nothing_behind_is_not_charged_the_grace_period(self) -> None: + # Checking the group on every exit has to be free in the case that is + # every real build: an empty group is a signal-0 probe, not a wait, so + # a provider that exited with no descendants is finished the moment its + # status is read. + with tempfile.TemporaryDirectory() as directory: + started = time.monotonic() + lifecycle._run_contained( + [sys.executable, "-c", "raise SystemExit(0)"], + environment={"PATH": os.environ.get("PATH", "")}, + cwd=directory, + timeout=60.0, + ) + elapsed = time.monotonic() - started + self.assertLess( + elapsed, + lifecycle._TERMINATION_GRACE_SECONDS, + "a clean run waited out a grace period it had nothing to wait for", + ) + + +class BuildAndPublishTests(TemporaryWorkspace): + def test_manifest_binds_every_required_fact(self) -> None: + manifest = self.build() + commit, tree = lifecycle.resolve_revision(self.repository) + census = lifecycle.read_tracked_census(self.repository, commit) + self.assertEqual(manifest.commit, commit) + self.assertEqual(manifest.tree, tree) + self.assertEqual(manifest.provider, PIN.as_metadata()) + self.assertEqual(manifest.built_at, NOW.isoformat()) + self.assertEqual(manifest.tracked_files, census.file_count) + self.assertEqual(manifest.tracked_bytes, census.total_bytes) + self.assertEqual(manifest.census_digest, census.digest) + self.assertEqual(manifest.graph_digest, hashlib.sha256(b"graph-bytes").hexdigest()) + self.assertEqual(manifest.graph_bytes, len(b"graph-bytes")) + self.assertEqual(manifest.completeness, lifecycle.COMPLETE) + + def test_manifest_round_trips_through_validation(self) -> None: + manifest = self.build() + self.assertEqual(lifecycle.load_manifest(manifest.to_json()), manifest) + + def test_shareable_summary_carries_no_content_or_local_path(self) -> None: + summary = self.build().shareable_summary() + rendered = json.dumps(summary) + self.assertNotIn(str(self.root), rendered) + self.assertNotIn("VALUE = 1", rendered) + self.assertNotIn("scratch", rendered) + + def test_state_is_private_to_the_operator(self) -> None: + self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertEqual(stat.S_IMODE(state.path.stat().st_mode), 0o700) + generation = state.current_generation() + self.assertEqual(stat.S_IMODE((state.generations_path / generation).stat().st_mode), 0o700) + self.assertEqual(stat.S_IMODE(state.artifact_path(generation).stat().st_mode), 0o600) + + def test_refresh_publishes_a_new_immutable_generation(self) -> None: + first = self.build(keep_previous=True) + (self.repository / "README.md").write_text("# changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + second = self.build(indexer=recording_indexer(b"second-graph"), keep_previous=True) + self.assertNotEqual(first.generation, second.generation) + self.assertNotEqual(first.commit, second.commit) + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertEqual(state.current_generation(), second.generation) + # The superseded generation is untouched, not rewritten in place. + self.assertEqual(state.read_manifest(first.generation), first) + self.assertEqual(state.artifact_path(first.generation).read_bytes(), b"graph-bytes") + + def test_pruning_keeps_only_the_published_generation(self) -> None: + self.build() + (self.repository / "README.md").write_text("# changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + second = self.build() + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), [second.generation]) + + def test_pruning_happens_before_the_build_lock_is_released(self) -> None: + """Pruning after unlocking can delete a concurrent builder's generation. + + A builder that released the lock, paused, and only then pruned would + remove whatever a second builder published in the meantime -- or that + builder's staging directory -- leaving ``current`` naming a directory + that no longer exists, with both builds reporting success. + """ + order: list[str] = [] + prune, release = lifecycle.GraphStateRoot.prune, lifecycle._BuildLock.__exit__ + + def record_prune(state, *, keep): + order.append("prune") + return prune(state, keep=keep) + + def record_release(lock, *exception): + order.append("unlock") + return release(lock, *exception) + + with mock.patch.object(lifecycle.GraphStateRoot, "prune", record_prune), \ + mock.patch.object(lifecycle._BuildLock, "__exit__", record_release): + self.build() + self.assertEqual(order, ["prune", "unlock"]) + + def test_a_failed_build_publishes_nothing(self) -> None: + first = self.build() + + def failing(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: + raise ContextError("provider failed") + + with self.assertRaises(ContextError): + self.build(indexer=failing) + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertEqual(state.current_generation(), first.generation) + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), [first.generation]) + + def test_a_provider_that_writes_nothing_fails_closed(self) -> None: + def silent(request: lifecycle.IndexRequest) -> lifecycle.IndexResult: + return lifecycle.IndexResult() + + with self.assertRaises(ContextError): + self.build(indexer=silent) + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), []) + + def test_an_oversized_artifact_is_refused_before_publication(self) -> None: + original = lifecycle.MAX_ARTIFACT_BYTES + lifecycle.MAX_ARTIFACT_BYTES = 4 + try: + with self.assertRaises(ContextError): + self.build(indexer=recording_indexer(b"too-large-for-the-budget")) + finally: + lifecycle.MAX_ARTIFACT_BYTES = original + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), []) + + def test_state_is_refused_inside_a_git_repository(self) -> None: + with self.assertRaises(ContextError): + self.build(root=self.repository / ".code-mower-state") + + def test_state_is_refused_behind_a_symlink_into_a_repository(self) -> None: + """A lexical ancestor walk does not see the repository through a link. + + ``--state-dir /outside/link/state`` names no repository in its own + spelling, but ``/outside/link`` can point at a directory inside one, and + the ``O_NOFOLLOW`` opens only cover the final component of each + directory this lane creates. Resolved, the path is inside the checkout + and the artifacts would land in it. + """ + inside = self.repository / "subdir" + inside.mkdir() + outside = self.root / "outside" + outside.mkdir() + os.symlink(inside, outside / "link") + with self.assertRaises(ContextError): + self.build(root=outside / "link" / "graph-state") + self.assertFalse((inside / "graph-state").exists()) + + def test_an_ordinary_symlinked_ancestor_is_resolved_not_refused(self) -> None: + # Resolving, not rejecting: private roots legitimately sit behind + # links -- macOS reaches ``/tmp`` through a link into its ``private`` + # directory -- so a symlinked ancestor outside any repository must + # still build. + elsewhere = self.root / "elsewhere" + elsewhere.mkdir() + os.symlink(elsewhere, self.root / "linked-state") + manifest = self.build(root=self.root / "linked-state" / "graph") + self.assertEqual(manifest.completeness, lifecycle.COMPLETE) + + def test_an_ancestor_retargeted_after_the_check_does_not_move_the_state(self) -> None: + """Check and use travel the same path, so retargeting between them does nothing. + + Checking a resolved snapshot and then writing through the original + spelling is two different paths: ``/outside/link`` can resolve outside + every repository when it is checked and point into one by the time the + first directory is created. The state root is canonicalized once, at + construction, and every later open goes through that canonical path, so + a link swapped afterwards is no longer on the route. + """ + safe = self.root / "safe" + safe.mkdir() + outside = self.root / "outside" + outside.mkdir() + link = outside / "link" + os.symlink(safe, link) + state = lifecycle.GraphStateRoot(self.repository, root=link / "graph-state") + inside = self.repository / "subdir" + inside.mkdir() + link.unlink() + os.symlink(inside, link) + state.ensure() + self.assertTrue(state.path.is_relative_to(safe)) + self.assertTrue((safe / "graph-state" / "graph").is_dir()) + self.assertFalse((inside / "graph-state").exists()) + + def test_a_component_that_becomes_a_symlink_before_creation_is_refused(self) -> None: + """The half a canonical snapshot cannot cover: a component that is not there yet. + + Resolving the root at construction settles what its *existing* + ancestors mean. It cannot settle what a component nobody has created + yet will mean, and a recursive ``mkdir`` would follow whatever appears + there. Here the nested root is absent at construction and an ancestor + of it is created as a link into the repository before ``ensure``: the + creation must refuse rather than write state inside the repository. + """ + outside = self.root / "outside" + outside.mkdir() + absent = outside / "deep" / "state" + state = lifecycle.GraphStateRoot(self.repository, root=absent) + inside = self.repository / "subdir" + inside.mkdir() + os.symlink(inside, outside / "deep") + with self.assertRaises(ContextError): + state.ensure() + self.assertFalse((inside / "state").exists()) + + def test_a_precreated_root_behind_a_new_ancestor_symlink_is_refused(self) -> None: + """The bypass a deepest-existing-prefix walk leaves open. + + Finding the deepest component that exists and opening *that* with + ``O_NOFOLLOW`` refuses an ancestor link only while the final directory + is still absent -- because then the walk has to create it, one + component at a time. Pre-create the final directory behind the link and + the walk has nothing left to create: it opens the whole absolute base + in one call, ``O_NOFOLLOW`` clears the leaf it was pointed at, and every + ancestor above the leaf is resolved exactly as the planted link + intended. The previous test leaves that directory absent and so never + reaches this. + + Here the root is absent at construction, ``/subdir/state`` is + pre-created, and the missing ancestor becomes a link to + ``/subdir``. The walk has to refuse on the ancestor itself. + """ + outside = self.root / "outside" + outside.mkdir() + absent = outside / "deep" / "state" + state = lifecycle.GraphStateRoot(self.repository, root=absent) + inside = self.repository / "subdir" + (inside / "state").mkdir(parents=True) + os.symlink(inside, outside / "deep") + # The leaf really is a directory and really is not a link, so nothing + # about the final component can catch this. + self.assertTrue(absent.is_dir()) + with self.assertRaises(ContextError): + state.ensure() + self.assertFalse((inside / "state" / "graph").exists()) + self.assertEqual(list((inside / "state").iterdir()), []) + + def test_an_ancestor_swapped_before_an_operation_refuses_it(self) -> None: + """A swapped ancestor is refused on the walk, not written through. + + Every mutation opens its directory by walking from ``/`` with + ``O_NOFOLLOW`` on each component, so an ancestor that has become a + symlink is refused where it is met rather than resolved into whatever + it points at. + """ + elsewhere = self.root / "elsewhere" + elsewhere.mkdir() + home = self.root / "home" + home.mkdir() + state = lifecycle.GraphStateRoot(self.repository, root=home / "st") + state.ensure() + decoy = self.root / "decoy" + os.rename(home, decoy) + os.symlink(elsewhere, home) + # Shaped like real state, so the refusal has to come from the walk + # rather than from the tree being absent through the link. + (elsewhere / "st" / "graph" / state.workspace / "generations").mkdir( + mode=0o700, parents=True + ) + with self.assertRaises(ContextError): + state.prune(keep=None) + with self.assertRaises(ContextError): + state.remove_all() + self.assertTrue((elsewhere / "st" / "graph" / state.workspace).is_dir()) + + def test_a_swap_after_the_walk_cannot_redirect_a_removal(self) -> None: + """The half a recheck cannot cover: a swap at the operation boundary. + + Revalidating the base immediately before ``shutil.rmtree(self.path)`` + reads as safe and is not. The recheck resolves the spelling once and + the removal resolves it again, so an ancestor swapped between those two + resolutions sends the whole recursive delete somewhere the check never + saw -- and the window is real however small, because the attacker picks + the instant. + + Here the swap is made deterministic at exactly that boundary: the + moment the walk has returned its descriptor and the deletion is about + to begin, ``holder`` is renamed aside and replaced with a link to a + private victim tree of the same shape. A path-based removal deletes the + victim. A descriptor-relative one deletes what the descriptor really + names, because there is no second resolution left to hijack. + """ + holder = self.root / "holder" + holder.mkdir() + state = lifecycle.GraphStateRoot(self.repository, root=holder / "state") + state.ensure() + victim = self.root / "victim" + target = victim / "state" / "graph" / state.workspace / "generations" + target.mkdir(mode=0o700, parents=True) + secret = target / "secret" + secret.write_bytes(b"not this process's to delete") + decoy = self.root / "decoy" + original = state._open_owned + + def swap_then_hand_over(*, depth: int): + handle = original(depth=depth) + os.rename(holder, decoy) + os.symlink(victim, holder) + return handle + + state._open_owned = swap_then_hand_over + self.assertTrue(state.remove_all()) + self.assertTrue(secret.exists()) + self.assertTrue((victim / "state" / "graph" / state.workspace).is_dir()) + self.assertFalse((decoy / "state" / "graph" / state.workspace).exists()) + + def test_a_symlink_inside_the_tree_is_unlinked_rather_than_followed(self) -> None: + """A removal walks its own tree without leaving it. + + A link planted among the generations names a path outside the state + root; deleting through it would delete the target. It is removed as the + link it is. + """ + outside = self.root / "outside" + outside.mkdir() + keepsake = outside / "keepsake" + keepsake.write_bytes(b"outside the state root") + state = lifecycle.GraphStateRoot(self.repository, root=self.root / "st") + state.ensure() + planted = state.generations_path / "planted" + os.symlink(outside, planted) + self.assertTrue(state.remove_all()) + self.assertFalse(state.path.exists()) + self.assertTrue(keepsake.exists()) + + def test_a_manifest_no_reader_could_load_publishes_nothing(self) -> None: + """Publication validates the whole manifest before it touches state. + + A manifest that this process can write but ``load_manifest`` refuses + would otherwise become ``current``, prune the generation that worked, + and read back ``invalid`` on the next status -- a build reporting + success while destroying the only usable graph. + """ + first = self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + unreadable = dataclasses.replace( + first, + generation=uuid.uuid4().hex, + skipped_paths=lifecycle.MAX_SKIPPED_PATHS + 1, + ) + with self.assertRaises(ContextError): + state.publish(unreadable, b"graph-bytes") + self.assertEqual(state.current_generation(), first.generation) + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), [first.generation]) + self.assertEqual(state.read_manifest(first.generation), first) + + +class StatusFailsClosedTests(TemporaryWorkspace): + def test_a_fresh_build_is_current(self) -> None: + manifest = self.build() + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(status.state, "current") + self.assertTrue(status.usable) + self.assertEqual(status.manifest, manifest) + + def test_no_state_is_absent_and_unusable(self) -> None: + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(status.state, "absent") + self.assertFalse(status.usable) + + def test_a_new_commit_makes_the_graph_stale(self) -> None: + self.build() + (self.repository / "README.md").write_text("# changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(status.state, "stale") + self.assertFalse(status.usable) + + def test_a_generation_pruned_mid_read_is_read_again(self) -> None: + # Readers take no lock, so a refresh can publish and prune between the + # pointer read and the validation of what it named. The failure that + # produces is about a directory a healthy build superseded, not about + # the graph the operator has. + first = self.build() + second = self.build(indexer=recording_indexer(b"second-graph")) + raced = lifecycle.GenerationStatus( + state="invalid", + generation=first.generation, + detail="local graph generation is missing its manifest", + ) + real, attempts = lifecycle._status_once, [] + + def once(state, repository, **keywords): + attempts.append(1) + return raced if len(attempts) == 1 else real(state, repository, **keywords) + + with mock.patch.object(lifecycle, "_status_once", once): + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(len(attempts), 2) + self.assertTrue(status.usable) + self.assertEqual(status.generation, second.generation) + + def test_a_verdict_about_the_published_generation_is_not_retried(self) -> None: + # The retry exists for a moved pointer only. A genuinely corrupt + # current generation is reported on the first read, not polled. + manifest = self.build() + artifact = lifecycle.GraphStateRoot(self.repository, root=self.state).artifact_path(manifest.generation) + artifact.write_bytes(b"tampered!!!") + real, attempts = lifecycle._status_once, [] + + def once(state, repository, **keywords): + attempts.append(1) + return real(state, repository, **keywords) + + with mock.patch.object(lifecycle, "_status_once", once): + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(status.state, "corrupt") + self.assertEqual(len(attempts), 1) + + def test_a_tampered_artifact_is_corrupt(self) -> None: + manifest = self.build() + artifact = lifecycle.GraphStateRoot(self.repository, root=self.state).artifact_path(manifest.generation) + artifact.write_bytes(b"tampered!!!") # same length, different content + status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual(status.state, "corrupt") + self.assertFalse(status.usable) + + def test_a_truncated_artifact_is_corrupt(self) -> None: + manifest = self.build() + artifact = lifecycle.GraphStateRoot(self.repository, root=self.state).artifact_path(manifest.generation) + artifact.write_bytes(b"short") + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "corrupt") + + def test_an_oversized_artifact_is_refused_on_read(self) -> None: + manifest = self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + artifact = state.artifact_path(manifest.generation) + original = lifecycle.MAX_ARTIFACT_BYTES + lifecycle.MAX_ARTIFACT_BYTES = 4 + try: + self.assertIn( + lifecycle.graph_status(self.repository, root=self.state).state, + ("corrupt", "oversized", "invalid"), + ) + finally: + lifecycle.MAX_ARTIFACT_BYTES = original + self.assertTrue(artifact.exists()) + + def test_a_partial_build_is_unusable_by_default(self) -> None: + self.build(indexer=recording_indexer(completeness=lifecycle.PARTIAL)) + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "partial") + allowed = lifecycle.graph_status(self.repository, root=self.state, require_complete=False) + self.assertEqual(allowed.state, "current") + + def test_a_corrupt_manifest_is_invalid(self) -> None: + manifest = self.build() + path = ( + lifecycle.GraphStateRoot(self.repository, root=self.state).generations_path + / manifest.generation + / lifecycle.MANIFEST_NAME + ) + path.write_text("{not json", encoding="utf-8") + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + + def test_a_manifest_missing_its_revision_binding_is_invalid(self) -> None: + manifest = self.build() + payload = manifest.to_json() + payload.pop("tree") + path = ( + lifecycle.GraphStateRoot(self.repository, root=self.state).generations_path + / manifest.generation + / lifecycle.MANIFEST_NAME + ) + path.write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + + def test_a_manifest_relabelled_to_another_generation_is_invalid(self) -> None: + manifest = self.build() + payload = manifest.to_json() + payload["generation"] = "f" * 32 + path = ( + lifecycle.GraphStateRoot(self.repository, root=self.state).generations_path + / manifest.generation + / lifecycle.MANIFEST_NAME + ) + path.write_text(json.dumps(payload), encoding="utf-8") + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + + def test_a_corrupt_pointer_is_invalid(self) -> None: + self.build() + (lifecycle.GraphStateRoot(self.repository, root=self.state).path / lifecycle.CURRENT_NAME).write_text( + "../../elsewhere\n", encoding="utf-8" + ) + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + + def test_group_readable_state_is_invalid(self) -> None: + """State loosened after the fact fails closed rather than being used.""" + self.build() + path = lifecycle.GraphStateRoot(self.repository, root=self.state).path + path.chmod(0o750) + try: + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + finally: + path.chmod(0o700) + + def test_a_group_readable_artifact_is_invalid(self) -> None: + manifest = self.build() + artifact = lifecycle.GraphStateRoot(self.repository, root=self.state).artifact_path(manifest.generation) + artifact.chmod(0o640) + try: + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "invalid") + finally: + artifact.chmod(0o600) + + +class GitBoundaryTests(TemporaryWorkspace): + """The other half of the offline boundary: Git's own reads. + + The provider runs inside a sandbox, but the census reader and the blob + materializer are Git children of this process, outside it. In a partial + clone their reads can fetch missing objects from a remote, so the boundary + has to cover them too. + """ + + def commit(self) -> str: + return lifecycle.resolve_revision(self.repository)[0] + + def test_git_children_get_no_lazy_fetch_and_no_transport(self) -> None: + environment = lifecycle.git_environment() + self.assertEqual(environment["GIT_NO_LAZY_FETCH"], "1") + # Set but empty: git reads the variable as the complete list of + # permitted transports, and an empty list permits none. + self.assertEqual(environment["GIT_ALLOW_PROTOCOL"], "") + self.assertEqual(environment["GIT_CONFIG_NOSYSTEM"], "1") + self.assertEqual(environment["GIT_CONFIG_GLOBAL"], os.devnull) + self.assertEqual(environment["GIT_CONFIG_SYSTEM"], os.devnull) + + def test_git_children_ignore_replacement_objects(self) -> None: + self.assertEqual(lifecycle.git_environment()["GIT_NO_REPLACE_OBJECTS"], "1") + + def test_a_replacement_object_cannot_substitute_committed_content(self) -> None: + """``refs/replace`` changes what a read returns, not what a commit names. + + Every ordinary Git read honours a replacement, so a census and a + materialization would bind bytes the recorded commit and tree do not + contain -- and deleting the replacement afterwards would leave + ``graph_status`` still reporting ``current``, because it compares object + names and nothing else. The unguarded read is asserted first so this + cannot pass by the replacement quietly not applying. + """ + tracked = "example_pkg/config.py" + committed = (self.repository / tracked).read_text(encoding="utf-8") + blob = git(self.repository, "rev-parse", f"HEAD:{tracked}").strip() + decoy = self.root / "decoy.py" + decoy.write_text("VALUE = 'substituted'\n", encoding="utf-8") + substitute = git(self.repository, "hash-object", "-w", str(decoy)).strip() + git(self.repository, "replace", blob, substitute) + self.assertEqual(git(self.repository, "cat-file", "blob", blob), decoy.read_text(encoding="utf-8")) + + census = lifecycle.read_tracked_census(self.repository, self.commit()) + entry = next(item for item in census.entries if item.path == tracked) + # ``ls-tree --long`` reports the replacement's size while still naming + # the original blob, so the census is bound too, not only the content. + self.assertEqual(entry.size, len(committed.encode("utf-8"))) + destination = self.root / "materialized" + lifecycle.materialize_tracked_files(self.repository, census, destination) + self.assertEqual((destination / tracked).read_text(encoding="utf-8"), committed) + + def test_git_children_inherit_no_ambient_secret(self) -> None: + with mock.patch.dict(os.environ, {"AWS_SECRET_ACCESS_KEY": "not-a-real-secret"}): + self.assertNotIn("AWS_SECRET_ACCESS_KEY", lifecycle.git_environment()) + + def test_the_transport_denial_outranks_repository_local_configuration(self) -> None: + # Local configuration belongs to the untrusted checkout and is always + # read, so the denial has to travel on the command line, which is the + # only level above it. + self.assertIn("protocol.allow=never", lifecycle._GIT_SAFETY_OPTIONS) + + def test_a_full_clone_is_read_without_complaint(self) -> None: + lifecycle.refuse_lazy_object_fetch(self.repository) + self.assertTrue(lifecycle.read_tracked_census(self.repository, self.commit()).entries) + + def test_a_partial_clone_is_refused_before_its_tree_is_read(self) -> None: + census = lifecycle.read_tracked_census(self.repository, self.commit()) + git(self.repository, "config", "--local", "remote.origin.promisor", "true") + with self.assertRaises(ContextError): + lifecycle.read_tracked_census(self.repository, self.commit()) + with self.assertRaises(ContextError): + lifecycle.materialize_tracked_files(self.repository, census, self.root / "fresh") + self.assertFalse((self.root / "fresh").exists()) + + def test_a_partial_clone_build_publishes_nothing(self) -> None: + git(self.repository, "config", "--local", "remote.origin.partialclonefilter", "blob:none") + with self.assertRaises(ContextError): + self.build() + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "absent") + + def test_the_partial_clone_extension_is_refused_too(self) -> None: + # The other shape it takes: a repository-format extension, with the + # version bump that makes git accept one. + git(self.repository, "config", "--local", "core.repositoryformatversion", "1") + git(self.repository, "config", "--local", "extensions.partialclone", "origin") + with self.assertRaises(ContextError): + lifecycle.refuse_lazy_object_fetch(self.repository) + + +class CheckoutIdentityTests(TemporaryWorkspace): + """One checkout is one workspace, however the command was spelled.""" + + def workspace(self, path: Path) -> str: + return lifecycle.GraphStateRoot(path, root=self.state).workspace + + def test_a_subdirectory_resolves_to_the_checkout_root(self) -> None: + subdirectory = self.repository / "example_pkg" + self.assertEqual(lifecycle.checkout_root(subdirectory), self.repository) + self.assertEqual(self.workspace(subdirectory), self.workspace(self.repository)) + + def test_a_symlinked_spelling_of_one_checkout_is_one_workspace(self) -> None: + link = self.root / "link-to-checkout" + link.symlink_to(self.repository) + self.assertEqual(self.workspace(link), self.workspace(self.repository)) + + def test_a_graph_built_at_the_root_is_current_from_a_subdirectory(self) -> None: + # The census is whole-tree, so a subdirectory asks about exactly the + # generation the root published. Reporting ``absent`` there would send + # a consumer to rebuild a graph it already has. + self.build() + status = lifecycle.graph_status(self.repository / "example_pkg", root=self.state) + self.assertEqual(status.state, "current") + self.assertTrue(status.usable) + + def test_a_build_from_a_subdirectory_publishes_the_checkouts_generation(self) -> None: + published = lifecycle.build_graph( + self.repository / "example_pkg", pin=PIN, indexer=recording_indexer(), + root=self.state, now=NOW, + ) + self.assertEqual( + list(lifecycle.iter_generations(self.repository, root=self.state)), + [published.generation], + ) + # The whole tree, not the subdirectory's slice of it: the census is + # read from the commit with ``--full-tree``, so where the command ran + # cannot narrow what the graph was built over. + self.assertEqual( + published.census_digest, + lifecycle.read_tracked_census(self.repository, published.commit).digest, + ) + root_status = lifecycle.graph_status(self.repository, root=self.state) + self.assertEqual((root_status.state, root_status.generation), ("current", published.generation)) + + def test_remove_from_a_subdirectory_deletes_the_checkouts_graph(self) -> None: + # The defect this covers reported success while deleting nothing: an + # operator clearing a graph from ``src/`` was told it was gone. + self.build() + self.assertTrue(lifecycle.remove_graph(self.repository / "example_pkg", root=self.state)) + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "absent") + + def test_doctor_from_a_subdirectory_sees_the_checkouts_generation(self) -> None: + self.build() + with stand_in_containment(("/sandbox",)): + report = lifecycle.doctor_report( + self.repository / "example_pkg", pin=PIN, root=self.state + ) + self.assertEqual(report["status"], "pass") + + def test_linked_worktrees_of_one_repository_keep_separate_state(self) -> None: + # Normalizing to a worktree root must not collapse two worktrees into + # one workspace: each holds its own revision, so a generation built for + # one is not a graph of the other's content. + other = self.root / "side" + git(self.repository, "worktree", "add", "-q", "-b", "side", str(other)) + other = other.resolve() + self.assertEqual(lifecycle.checkout_root(other), other) + self.assertNotEqual(self.workspace(other), self.workspace(self.repository)) + self.build() + self.assertEqual(lifecycle.graph_status(other, root=self.state).state, "absent") + # ...and a subdirectory of the linked worktree resolves to that + # worktree, not to the repository it was created from. + self.assertEqual(lifecycle.checkout_root(other / "example_pkg"), other) + + def test_two_unrelated_checkouts_keep_separate_state(self) -> None: + other = self.root / "other" + other.mkdir() + self.assertNotEqual(self.workspace(other), self.workspace(self.repository)) + + def test_a_path_outside_any_repository_keeps_its_resolved_path(self) -> None: + # Identity stays defined for a path Git cannot place. The verbs that + # need Git fail on their own terms; naming state must not be the step + # that breaks, or ``remove`` could not clean up after one. + outside = self.root / "not-a-repository" + outside.mkdir() + self.assertEqual(lifecycle.checkout_root(outside), outside) + self.assertEqual( + lifecycle.checkout_root(self.root / "absent"), (self.root / "absent").resolve() + ) + + +class RemoveTests(TemporaryWorkspace): + def test_remove_takes_the_build_lock(self) -> None: + # A removal running beside a build deletes its sources, its output and + # its generations; the builder then either fails or recreates state + # that ``remove`` has already reported as gone. + self.build() + taken: list[str] = [] + lock = lifecycle.GraphStateRoot.lock + + def record_lock(state): + taken.append("lock") + return lock(state) + + with mock.patch.object(lifecycle.GraphStateRoot, "lock", record_lock): + self.assertTrue(lifecycle.remove_graph(self.repository, root=self.state)) + self.assertEqual(taken, ["lock"]) + + def test_the_lock_survives_the_removal_it_serializes(self) -> None: + # The inode a waiting builder is blocked on must still be there when + # the remover lets go of it. A lock file inside the deleted tree would + # be unlinked mid-removal and the next builder would lock a new one. + self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertFalse(state.lock_path.is_relative_to(state.path)) + before = state.lock_path.stat().st_ino + self.assertTrue(lifecycle.remove_graph(self.repository, root=self.state)) + self.assertTrue(state.lock_path.exists()) + self.assertEqual(state.lock_path.stat().st_ino, before) + + def test_the_retained_lock_carries_nothing_and_stays_private(self) -> None: + self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + lifecycle.remove_graph(self.repository, root=self.state) + self.assertEqual(state.lock_path.read_bytes(), b"") + self.assertEqual(stat.S_IMODE(state.lock_path.stat().st_mode), 0o600) + + def test_removing_nothing_creates_nothing(self) -> None: + # A removal on an installation that never opted in must not bring a + # private state tree into existence just to report that it is empty. + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertFalse(lifecycle.remove_graph(self.repository, root=self.state)) + self.assertFalse(state.path.exists()) + self.assertFalse(state.lock_path.exists()) + + def test_remove_deletes_every_generation(self) -> None: + self.build() + state = lifecycle.GraphStateRoot(self.repository, root=self.state) + self.assertTrue(lifecycle.remove_graph(self.repository, root=self.state)) + self.assertFalse(state.path.exists()) + self.assertEqual(lifecycle.graph_status(self.repository, root=self.state).state, "absent") + + def test_remove_is_idempotent(self) -> None: + self.assertFalse(lifecycle.remove_graph(self.repository, root=self.state)) + + def test_remove_refuses_state_that_is_not_private(self) -> None: + self.build() + path = lifecycle.GraphStateRoot(self.repository, root=self.state).path + path.chmod(0o755) + try: + with self.assertRaises(ContextError): + lifecycle.remove_graph(self.repository, root=self.state) + self.assertTrue(path.exists()) + finally: + path.chmod(0o700) + + +class DoctorTests(TemporaryWorkspace): + def test_an_unconfigured_installation_skips_rather_than_fails(self) -> None: + report = lifecycle.doctor_report(self.repository, pin=None, root=self.state) + self.assertEqual(report["status"], "skip") + self.assertEqual({check["status"] for check in report["checks"]}, {"skip"}) + + def test_a_healthy_build_passes(self) -> None: + self.build() + # Isolation is a property of the host, not of this build; a host that + # offers a sandbox is the healthy case being described here. + with stand_in_containment(("/sandbox",)): + report = lifecycle.doctor_report(self.repository, pin=PIN, root=self.state) + self.assertEqual(report["status"], "pass") + + def test_a_host_that_cannot_contain_a_provider_fails_doctor(self) -> None: + self.build() + with no_containment(): + report = lifecycle.doctor_report(self.repository, pin=PIN, root=self.state) + self.assertEqual(report["status"], "fail") + isolation = [check for check in report["checks"] if check["check"] == "context-graph-isolation"] + self.assertEqual([check["status"] for check in isolation], ["fail"]) + + def test_isolation_is_not_asked_about_when_nothing_is_pinned(self) -> None: + report = lifecycle.doctor_report(self.repository, pin=None, root=self.state) + isolation = [check for check in report["checks"] if check["check"] == "context-graph-isolation"] + self.assertEqual([check["status"] for check in isolation], ["skip"]) + + def test_a_stale_graph_fails_doctor(self) -> None: + self.build() + (self.repository / "README.md").write_text("# changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + report = lifecycle.doctor_report(self.repository, pin=PIN, root=self.state) + self.assertEqual(report["status"], "fail") + + def test_doctor_output_carries_no_indexed_content(self) -> None: + self.build() + rendered = json.dumps(lifecycle.doctor_report(self.repository, pin=PIN, root=self.state)) + self.assertNotIn("VALUE = 1", rendered) + self.assertNotIn("not-a-real-secret", rendered) + + +class CommandTests(TemporaryWorkspace): + def pin_file(self) -> Path: + path = self.root / "pin.json" + path.write_text(json.dumps(PIN.as_metadata()), encoding="utf-8") + return path + + def indexer_script(self, *, complete: bool = True, **installed) -> Path: + """A stand-in for a pinned provider CLI, so no package is required. + + It answers to ``extract`` and writes its state into the directory it + was run in, which is the contract ``docs/graphify-evaluation.md`` + records for the evaluated release. + + Installed the way a pinned release really is -- a console script in a + virtual environment beside the ``.dist-info`` its installer wrote -- + because that is what both the containment boundary and the pin check + read before a build runs anything. A loose script in a temporary + directory is refused, and these end-to-end tests should run against the + arrangement an operator is actually told to pin into rather than one + the rules would reject. + """ + return install_provider( + self.root / ("provider-venv" if complete else "provider-venv-partial"), + script="fake-indexer", + body=( + "#!/bin/sh\n" + '[ "$1" = "extract" ] || exit 64\n' + "mkdir -p .graphify\n" + "printf graph-bytes > .graphify/graph.bin\n" + 'printf \'{"complete": %s, "code_files": 1, "requeued": 0}\' ' + f"'{'true' if complete else 'false'}' > .graphify/manifest.json\n" + ), + **installed, + ) + + def run_command(self, *arguments: str) -> tuple[int, str]: + from contextlib import redirect_stdout + from io import StringIO + + buffer = StringIO() + with redirect_stdout(buffer): + code = command.main(list(arguments)) + return code, buffer.getvalue() + + def base(self) -> list[str]: + return ["--repo-path", str(self.repository), "--state-dir", str(self.state), "--json"] + + def test_build_status_refresh_remove_round_trip(self) -> None: + # The only test that launches a provider for real, so it is also the + # only one that needs the host to offer the sandbox a build requires. + require_containment(self) + pin, indexer = str(self.pin_file()), str(self.indexer_script()) + code, output = self.run_command("build", *self.base(), "--pin-file", pin, "--indexer", indexer) + self.assertEqual(code, 0, output) + published = json.loads(output) + self.assertEqual(published["status"], "published") + + code, output = self.run_command("status", *self.base()) + self.assertEqual(code, 0, output) + self.assertTrue(json.loads(output)["usable"]) + + # A second ``build`` refuses; ``refresh`` is the explicit rebuild verb. + code, _ = self.run_command("build", *self.base(), "--pin-file", pin, "--indexer", indexer) + self.assertEqual(code, 1) + code, output = self.run_command("refresh", *self.base(), "--pin-file", pin, "--indexer", indexer) + self.assertEqual(code, 0, output) + self.assertNotEqual(json.loads(output)["generation"], published["generation"]) + + code, output = self.run_command("remove", *self.base()) + self.assertEqual(code, 0, output) + self.assertTrue(json.loads(output)["removed"]) + code, output = self.run_command("status", *self.base()) + self.assertEqual(code, 1) + self.assertEqual(json.loads(output)["state"], "absent") + + def test_the_verbs_address_one_checkout_from_any_directory_inside_it(self) -> None: + """``--repo-path`` defaults to the current directory, so this is the + common case rather than an exotic one: the graph is built once at the + root and then asked about from wherever the operator is working. + + Built through the injected indexer rather than the launcher, because + what is under test is which state the verbs address, not containment. + """ + self.build() + inside = ["--repo-path", str(self.repository / "example_pkg"), + "--state-dir", str(self.state), "--json"] + code, output = self.run_command("status", *inside) + self.assertEqual(code, 0, output) + self.assertTrue(json.loads(output)["usable"]) + code, output = self.run_command("doctor", *inside) + self.assertEqual(code, 0, output) + generation = [check for check in json.loads(output)["checks"] + if check["check"] == "context-graph-generation"] + self.assertEqual([check["status"] for check in generation], ["pass"]) + code, output = self.run_command("remove", *inside) + self.assertEqual(code, 0, output) + self.assertTrue(json.loads(output)["removed"]) + # Removed the checkout's graph, not a namesake of its own: the root + # now reports absent too. + code, output = self.run_command("status", *self.base()) + self.assertEqual(code, 1) + self.assertEqual(json.loads(output)["state"], "absent") + + def test_a_provider_that_admits_an_incomplete_run_is_not_usable(self) -> None: + # End to end through the real launcher: the provider exits zero and + # writes state, and the build is still refused because its own report + # denies completion. Exit status is not completion evidence. + require_containment(self) + code, output = self.run_command( + "build", *self.base(), + "--pin-file", str(self.pin_file()), + "--indexer", str(self.indexer_script(complete=False)), + ) + # The build says so itself. Reporting ``current`` here and ``partial`` + # one command later would describe an unusable generation as usable for + # exactly as long as it took to ask again. + self.assertEqual(code, 1, output) + published = json.loads(output) + self.assertFalse(published["usable"]) + self.assertEqual(published["completeness"], lifecycle.PARTIAL) + code, output = self.run_command("status", *self.base()) + self.assertEqual(code, 1) + self.assertEqual(json.loads(output)["state"], "partial") + + def test_an_incomplete_build_is_printed_as_partial(self) -> None: + require_containment(self) + arguments = [argument for argument in self.base() if argument != "--json"] + code, output = self.run_command( + "build", *arguments, + "--pin-file", str(self.pin_file()), + "--indexer", str(self.indexer_script(complete=False)), + ) + self.assertEqual(code, 1, output) + self.assertIn("Local graph: partial", output) + self.assertNotIn("Local graph: current", output) + + def test_status_reports_stale_with_a_nonzero_exit(self) -> None: + self.build() + (self.repository / "README.md").write_text("# changed\n", encoding="utf-8") + git(self.repository, "commit", "-q", "-am", "change") + code, output = self.run_command("status", *self.base()) + self.assertEqual(code, 1) + self.assertEqual(json.loads(output)["state"], "stale") + + def test_build_without_a_pin_is_refused(self) -> None: + code, _ = self.run_command("build", *self.base(), "--indexer", str(self.indexer_script())) + self.assertEqual(code, 1) + self.assertEqual(list(lifecycle.iter_generations(self.repository, root=self.state)), []) + + def test_remove_hides_the_private_path_unless_asked(self) -> None: + self.build() + _, output = self.run_command("remove", *self.base()) + self.assertNotIn(str(self.state), output) + + def test_doctor_reports_an_unconfigured_installation(self) -> None: + code, output = self.run_command("doctor", *self.base()) + self.assertEqual(code, 0, output) + self.assertEqual(json.loads(output)["status"], "skip") + + def test_command_is_registered_on_the_cli(self) -> None: + from code_mower import cli + + self.assertIs(cli.COMMAND_HANDLERS["context-graph"], command.main) + self.assertIn("context-graph", cli.COMMAND_DESCRIPTIONS) + + +if __name__ == "__main__": # pragma: no cover - direct invocation + unittest.main() diff --git a/tests/test_release_hygiene.py b/tests/test_release_hygiene.py index cdb491fc..c55240f6 100644 --- a/tests/test_release_hygiene.py +++ b/tests/test_release_hygiene.py @@ -375,6 +375,7 @@ def test_cli_command_registry_is_single_source_of_truth(self) -> None: "cloud", "config", "context", + "context-graph", "context-packs", "controller", "coderabbit-cli",