From 28e5a1418d00aee1357ed876d98fcb4c336c2e5f Mon Sep 17 00:00:00 2001 From: hypatia Date: Sat, 26 Sep 2026 14:11:15 +0000 Subject: [PATCH 1/6] fix(ci): repin codeql-action off the v4.38.1 startup-killer; add estate sweeps Two poisoned pins in this repository were left by the 2026-09-22 rollback, which relabelled the pin without re-pinning it: github/codeql-action@1c5b6756 (annotated `# v4.38.0`) is the commit GitHub rejects at workflow start-up, so CodeQL and the security scan die with zero jobs wherever it appears. actions.lock already carries the correct b96794f0 pin; the workflows did not. This makes the workflows agree with the lock. Adds the estate machinery the September incident showed was missing: lib/rules/pin_integrity.ex PI001-PI005 + mechanical substitution lib/rules/pr_automerge.ex PA001-PA006 + decision manifests test/rules/*.exs the incident's cases, as tests scripts/sweeps/estate-*.sh pin repair, PR disposition, absence intake, estate statistics pr-automerge-policy.json the single policy both readers consume docs/operations/estate-automerge.adoc the handoff document The rule the whole design turns on: pin identity is the SHA, never the inline comment. A version-string check sees nothing wrong with `1c5b6756 # v4.38.0`, which is exactly how the poison survived the first rollback and came back in 25 of the estate's 34 open dependabot PRs. Not a self-approved change: rules, bot directives and this repository are under the reflexivity guard (NA-005). Review before merge. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 +- .github/workflows/security-policy.yml | 12 +- .gitignore | 4 + .../pr-automerge-policy.json | 245 +++++++ docs/operations/estate-automerge.adoc | 225 ++++++ lib/rules/pin_integrity.ex | 694 ++++++++++++++++++ lib/rules/pr_automerge.ex | 517 +++++++++++++ scripts/sweeps/estate-absence-intake.sh | 195 +++++ scripts/sweeps/estate-pin-integrity.sh | 307 ++++++++ scripts/sweeps/estate-pr-automerge.sh | 561 ++++++++++++++ scripts/sweeps/estate-stats.sh | 403 ++++++++++ test/rules/pin_integrity_test.exs | 393 ++++++++++ test/rules/pr_automerge_test.exs | 358 +++++++++ 13 files changed, 3910 insertions(+), 8 deletions(-) create mode 100644 .machine_readable/merge-orchestration/pr-automerge-policy.json create mode 100644 docs/operations/estate-automerge.adoc create mode 100644 lib/rules/pin_integrity.ex create mode 100644 lib/rules/pr_automerge.ex create mode 100755 scripts/sweeps/estate-absence-intake.sh create mode 100755 scripts/sweeps/estate-pin-integrity.sh create mode 100755 scripts/sweeps/estate-pr-automerge.sh create mode 100755 scripts/sweeps/estate-stats.sh create mode 100644 test/rules/pin_integrity_test.exs create mode 100644 test/rules/pr_automerge_test.exs diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7fa8f860..6395a96f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -47,12 +47,12 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: languages: ${{ matrix.language }} build-mode: ${{ matrix.build-mode }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/security-policy.yml b/.github/workflows/security-policy.yml index 70d1b6d9..b32fae8b 100644 --- a/.github/workflows/security-policy.yml +++ b/.github/workflows/security-policy.yml @@ -248,13 +248,13 @@ jobs: uses: actions/checkout@v7.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: languages: ${{ matrix.language }} queries: security-extended,security-and-quality - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: category: "/language:${{matrix.language}}" @@ -328,7 +328,7 @@ jobs: severity: 'CRITICAL,HIGH' - name: Upload Trivy scan results - uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) if: always() with: sarif_file: 'trivy-results.sarif' @@ -580,7 +580,7 @@ jobs: ignore-unfixed: true - name: Upload Trivy SARIF results - uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) if: always() with: sarif_file: 'trivy-${{ steps.image.outputs.name }}.sarif' @@ -818,13 +818,13 @@ jobs: uses: actions/checkout@v7.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: languages: ${{ matrix.language }} queries: security-extended,security-and-quality - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) with: category: "/language:${{matrix.language}}" diff --git a/.gitignore b/.gitignore index 00d9f125..5dc6e554 100644 --- a/.gitignore +++ b/.gitignore @@ -152,3 +152,7 @@ dist/ # (mode 160000 with no .gitmodules), giving every clone four broken # submodule pointers. Ignored so it cannot recur. .claude/worktrees/ + +# Sweep cache/artifacts (estate-pr-automerge.sh). Rebuildable; never a source +# file. +.pr-automerge/ diff --git a/.machine_readable/merge-orchestration/pr-automerge-policy.json b/.machine_readable/merge-orchestration/pr-automerge-policy.json new file mode 100644 index 00000000..89e2a09f --- /dev/null +++ b/.machine_readable/merge-orchestration/pr-automerge-policy.json @@ -0,0 +1,245 @@ +{ + "$schema": "https://github.com/hyperpolymath/hypatia/blob/main/docs/design/merge-orchestration/schemas/pr-automerge-policy.schema.json", + "$comment": "SPDX-License-Identifier: MPL-2.0 — Single source of truth for estate PR automerge + pin integrity. Read by BOTH the Elixir brain (Hypatia.Automerge.Policy) and the token-bearing actuator (scripts/estate/pr-automerge.sh). Two independent readers, one policy file: the actuator never trusts the brain's verdict, it re-derives it from the same data.", + "version": "1.0.0", + "updated": "2026-09-26", + "owner": "hyperpolymath", + "description": "Declarative policy for the 'safer core' of hypatia: unambiguous bumps/chores/pins get merged and their branches deleted; poisoned pins are excised or vetoed; everything else is flagged. Written after the 2026-09-26 finding that 108 workflow files across 84 hyperpolymath repos (and 23 in metadatastician) carry the codeql-action v4.38.1 commit labelled as v4.38.0.", + + "pin_denylist": [ + { + "id": "PIN-001", + "action": "github/codeql-action", + "ecosystem": "github-actions", + "severity": "critical", + "blocked_versions": ["4.38.1"], + "blocked_shas": ["1c5b675653bb5c22dbe9b12b556ec555138e09fd"], + "blocked_refs": ["v4.38.1"], + "reason": "GitHub rejects this commit at workflow start-up: `startup_failure`, zero jobs, no logs, wherever it is used. Kills CodeQL, Hypatia Security Scan and Scorecard on every repo it reaches.", + "known_good_version": "4.38.0", + "known_good_sha": "b96794f015dfd88f77b49b1c93e0fa7110f94c63", + "replacement": "pin_to_known_good", + "evidence": [ + "hyperpolymath/nexia-list#100 — rollback, with the startup_failure signature reproduced across two workflows", + "hyperpolymath/standards#1037 — population + the dependabot.yml ignore-rule bypass", + "hyperpolymath/standards#1005 AC2 — the 40-repo re-pin" + ], + "added": "2026-09-23", + "added_by": "hypatia/standards#1037", + "notes": "The denylist is keyed on SHA as well as version BECAUSE the estate contains files where this SHA is annotated `# v4.38.0`. A version-string-only check sees nothing wrong. This is the whole reason the rule exists at file level rather than on the PR title." + }, + { + "id": "PIN-002", + "action": "dtolnay/rust-toolchain", + "ecosystem": "github-actions", + "severity": "high", + "blocked_versions": [], + "blocked_shas": [], + "blocked_refs": [], + "reason": "Moving ref (`stable`) must never be SHA-pinned in actions.lock: the pin stops matching the ref on every Rust release and every job using it dies at `Set up job`.", + "known_good_version": null, + "known_good_sha": null, + "replacement": "unpin_moving_ref", + "evidence": ["hypatia/docs/DEBT-REGISTER.adoc CI-1 (run 31170993296, job 92842543720)"], + "added": "2026-09-26", + "added_by": "hypatia debt register CI-1", + "notes": "Structural cure, not a re-pin: drop the entry from actions.lock rather than pinning a new SHA." + } + ], + + "moving_refs_never_lockable": [ + "stable", + "beta", + "nightly", + "latest", + "main", + "master", + "HEAD" + ], + + "safe_change_classes": { + "bump_ci_patch_minor": { + "description": "Dependabot/renovate bump of a github-actions dependency within the same major, touching only `uses:` lines.", + "route": "Patch-Bridge", + "method": "squash", + "pool": "P2", + "safety": "arm_auto", + "requires": ["author_is_dependency_bot", "semver_patch_or_minor", "no_denylisted_pin", "delta_is_pin_lines_only"] + }, + "bump_lockfile_only": { + "description": "Dependency bump whose diff is confined to lockfiles (Cargo.lock, mix.lock, package-lock.json, go.sum, flake.lock).", + "route": "Patch-Bridge", + "method": "squash", + "pool": "P2", + "safety": "arm_auto", + "requires": ["author_is_dependency_bot", "no_denylisted_pin", "all_files_are_lockfiles"] + }, + "chore_meta_only": { + "description": "Non-code chore: docs, licence headers, linguist exclusions, CODEOWNERS, issue templates, .gitattributes. No runtime path.", + "route": "rhodibot", + "method": "squash", + "pool": "P3", + "safety": "arm_auto", + "requires": ["no_denylisted_pin", "no_code_path_change", "no_workflow_semantic_change"] + }, + "pin_rollback_denylisted": { + "description": "A pin currently on the denylist is replaced by its known-good SHA, in place, with no other edit.", + "route": "Patch-Bridge", + "method": "squash", + "pool": "P1", + "safety": "arm_auto", + "requires": ["every_changed_line_is_a_denylisted_pin_site", "replacement_equals_known_good"] + } + }, + + "never_auto": [ + { + "id": "NA-001", + "match": "semver_major", + "reason": "Major bumps change behaviour by definition." + }, + { + "id": "NA-002", + "match": "security_advisory", + "reason": "Security updates are routed to the owner + panicbot; auto-merge of an advisory bump can be an Akerlof claim-grounder problem." + }, + { + "id": "NA-003", + "match": "new_or_removed_workflow", + "reason": "Adding or deleting a workflow is a change to what the estate does, not to which version of it runs." + }, + { + "id": "NA-004", + "match": "permissions_trigger_or_expression_change", + "reason": "Any delta to `permissions:`, `on:`, `if:`, `env:` or `secrets:` is semantic, even inside a workflow." + }, + { + "id": "NA-005", + "match": "touches_oracle", + "reason": "Reflexivity guard: changes to hypatia rules, bot_directives, pool/TRUST levels or the standards repo are proposed, never self-approved." + }, + { + "id": "NA-006", + "match": "human_authored_unreviewed", + "reason": "Automerge is for machine-authored chores. A human PR keeps its human reviewer." + } + ], + + "meta_guard": { + "comment": "Change-level is `meta` (and therefore safety=flag) whenever the delta reaches CI, the oracle, or policy. The single exemption is a pure pin substitution — the actuator must re-prove `pin_only` from the diff itself before honouring it.", + "meta_paths": [ + ".github/workflows/", + ".github/actions/", + "lib/rules/", + "lib/automerge/", + ".machine_readable/bot_directives/", + ".machine_readable/merge-orchestration/", + "stdlib/" + ], + "meta_repos": ["standards", "rsr-template-repo", "hypatia"], + "exemptions": [ + { + "id": "MGX-001", + "name": "pin_only_workflow_edit", + "description": "Every added/removed line in the diff is a `uses:` pin token substitution on a line that exists on both sides, the action name is unchanged, and the target is not denylisted.", + "requires_actuator_reproof": true + } + ] + }, + + "lockfile_names": [ + "Cargo.lock", + "mix.lock", + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", + "bun.lock", + "bun.lockb", + "go.sum", + "flake.lock", + "Manifest.toml", + "poetry.lock", + "Gemfile.lock", + "composer.lock" + ], + + "dependency_bots": [ + "dependabot[bot]", + "renovate[bot]", + "renovate-bot", + "app/dependabot", + "app/renovate" + ], + + "absence_rules": [ + { + "id": "ABS-001", + "field": "description", + "test": "empty_or_placeholder", + "severity": "high", + "auto_fixable": false, + "why": "A repository with no description is not discoverable and does not state its own purpose. Authoring one is a judgement call — hypatia must not invent it.", + "issue_title": "Repository description is empty", + "issue_body_template": "`.github` metadata for this repository has no description.\n\nThis is one of the RSR conformance absences that hypatia cannot fix unambiguously: a description has to say what the project *is*, which is a claim only the maintainer can make.\n\n**What would close this:** a one-line `description` in the repository settings (or a `README.adoc` first paragraph that can be lifted verbatim).\n\nFiled by `hypatia` (ABS-001) with the absence-intake run of {date}. Deduplicated by this marker: ``" + }, + { + "id": "ABS-002", + "field": "topics", + "test": "fewer_than_min", + "min": 7, + "severity": "high", + "auto_fixable": false, + "why": "RSR requires at least 7 topics: language, ecosystem, status, licence, domain, maturity and one distinguishing topic. Which seven is a judgement call about the project.", + "issue_title": "Repository topics are below the RSR minimum of 7", + "issue_body_template": "This repository has {topics_count} topic(s); the RSR standard requires at least 7.\n\nCurrent: {topics_list}\n\n**What would close this:** set at least 7 topics in repository settings covering language, ecosystem, status, licence, domain, maturity and one distinguishing topic.\n\nFiled by `hypatia` (ABS-002) with the absence-intake run of {date}. Deduplicated by this marker: ``" + }, + { + "id": "ABS-003", + "field": "chrome", + "test": "missing_required_file", + "required_files": ["LICENSE", "README.adoc", "SECURITY.adoc"], + "severity": "medium", + "auto_fixable": "seed_from_template", + "why": "Standard project chrome. A seed from rsr-template-repo is unambiguous; a missing licence *choice* is not, so LICENSES/ is seeded but never invented.", + "issue_title": "Standard project chrome is incomplete", + "issue_body_template": "Missing from the repository root: {missing_list}\n\nRSR template chrome is seeded, not invented: `hypatia` can copy the template skeleton, but the licence *choice* and the security *contact* need a human.\n\nFiled by `hypatia` (ABS-003) with the absence-intake run of {date}. Deduplicated by this marker: ``" + } + ], + + "intake_limits": { + "comment": "The 2026-09-26 complaint was that issues had become unreadable. These caps are the cure: intake is capped per run, deduplicated by an in-body marker, and upgradeable — a repeat finding comments on the existing issue instead of filing a new one.", + "max_new_issues_per_run": 25, + "max_new_issues_per_repo_per_run": 2, + "never_reopen_closed": true, + "comment_on_existing_instead_of_new": true, + "roll_up_when_class_exceeds": 20 + }, + + "stats_thresholds": { + "comment": "Live thresholds for the private-farm dashboard. A metric crosses into 'recent/live tracking' when it exceeds these; below them it is reported as a headline number only.", + "failing_tests_warn": 1, + "failing_tests_critical": 25, + "test_coverage_empties_warn": 11, + "test_coverage_empties_critical": 47, + "bench_coverage_empties_warn": 11, + "bench_coverage_empties_critical": 47, + "bench_over_limit_ratio_warn": 1.5, + "bench_over_limit_ratio_critical": 3.0, + "unmerged_pr_age_days_warn": 7, + "unmerged_pr_age_days_critical": 30, + "unmerged_pr_count_warn": 10, + "unmerged_pr_count_critical": 50, + "open_issue_count_warn": 250, + "open_issue_count_critical": 500 + }, + + "stats_output": { + "comment": "Where the dashboard artifact goes. The .git-private-farm repo is not readable or writable from the public estate — this is the contract it consumes.", + "artifact_name": "estate-stats.json", + "schema_version": "1.0.0", + "consumer_repo": "hyperpolymath/.git-private-farm", + "consumer_path": "metadatastician/berrywiki/data/estate-stats.json", + "producer_path": "docs/status/estate-stats.json", + "history_dir": "docs/status/history/" + } +} diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc new file mode 100644 index 00000000..53f3d631 --- /dev/null +++ b/docs/operations/estate-automerge.adoc @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Estate auto-merge: the pin-rollback path, the absence intake, and the stats artifact +:author: Jonathan D.A. Jewell +:date: 2026-09-26 + +== Why this document exists + +On 2026-09-26 the estate had 34 open pull requests, every one of them a +`chore(deps): bump …` from dependabot. Every one of them *looked* mergeable. +Twenty-four of them re-introduced `github/codeql-action` v4.38.1, the commit +GitHub rejects at workflow start-up (`startup_failure`, zero jobs, no logs) — +and two that had already merged (`nexia-list#107`, `dictask#65`) had left +their repositories red on `main`. + +The rollback had itself been applied as a *relabel*: files across the estate +still carry + + uses: github/codeql-action/init@1c5b6756… # v4.38.0 + +which is the poisoned commit wearing a correct-looking comment. A gate that +greps for the version string sees a correct pin. A gate that greps for the +commit sees it in 110 files across 85 repositories. + +This document describes the machinery that exists to make that class of +mistake impossible, and the exact limits of what hypatia is allowed to do +about it on its own. + +== The split: brain writes, actuator executes + +Two implementations read one policy file, deliberately: + +* **The brain** — `lib/rules/pr_automerge.ex` (`PA001`–`PA006`) and + `lib/rules/pin_integrity.ex` (`PI001`–`PI005`). Pure functions: a PR record + and the policy in, a decision out. No network, no clock, no repo handle. + These are what an offline classifier can run and what the unit tests + exercise (`test/rules/pr_automerge_test.exs`, `test/rules/pin_integrity_test.exs`). +* **The actuator** — `scripts/sweeps/estate-pr-automerge.sh`, + `scripts/sweeps/estate-pin-integrity.sh`. These hold the token, fetch the + live diff, and — before any write — *re-derive* the safety rules against + what the repository actually contains right now. + +The division is not a stylistic choice. The app token used by the shared CI +path can create issues and nothing else: no push, no branch create or delete, +no merge, no label. Merging and branch deletion therefore belong to a host +that owns a write token — the private farm's actuator — and it must never +trust the brain's claim that a change is safe. It re-proves it. + +The one exemption is `MGX-001`: a workflow edit is `meta` (owner review) — +change level `meta`, safety `flag` — *except* when every changed line is a +`uses:` pin token substitution on the same action, which both implementations +re-prove from the raw diff (`pin_only_edit?/3` / the shell's `nonpin_change` +counter). A new step, a changed `permissions:` block, a moved `if:` — any of +those fails the proof and the exemption is void. + +== The policy is the single source of truth + +`.machine_readable/merge-orchestration/pr-automerge-policy.json` (v1.0.0) holds: + +* `pin_denylist` — `PIN-001` (`github/codeql-action`, blocked SHAs + `1c5b6756…` and version `4.38.1`, known-good `b96794f0…` / `4.38.0`) and + `PIN-002` (`dtolnay/rust-toolchain`); `moving_refs_never_lockable` + (`stable`, `latest`, `main`, …). +* `safe_change_classes` — `bump_ci_patch_minor`, `bump_lockfile_only`, + `chore_meta_only`, `pin_rollback_denylisted`. +* `never_auto` — `NA-001`…`NA-006`: semver major, security advisory, workflow + added/removed, `permissions:`/`on:`/`if:`/`env:`/`secrets:` deltas, + oracle/reflexivity files, human-authored PRs. +* `meta_guard` — the paths and repositories where `MGX-001` is the only way + past owner review. +* `absence_rules` — `ABS-001` (empty description), `ABS-002` (fewer than 7 + topics), `ABS-003` (repository chrome); `intake_limits` (25 new issues per + run, 2 per repository per run, roll up a class over 20). +* `stats_thresholds` / `stats_output` — the dashboard contract, below. + +If a rule changes, it changes here first. A rule that exists in only one of +the two engines is a bug. + +== The decision record + +Every examined PR gets one line in `pr-decisions.jsonl`: + +|=== +| Field | Values + +| `disposition` +| `auto_merge`, `excise_poison_then_merge`, `close_poison_only`, + `close_poison_and_majors`, `close_archived_repo`, `flag` + +| `blocked_by` +| `awaiting_required_checks`, `introduces_denylisted_pin`, + `introduces_denylisted_pin_and_major_bumps`, + `introduces_denylisted_pin_alongside_wanted_updates`, `major_version_delta`, + `version_claims_conflict`, `pin_delta_unresolvable`, + `dependency_manifest_not_lockfile`, `touches_licence_requires_owner_review`, + `repository_is_archived_cannot_merge`, `not_a_dependency_bot`, + `not_unambiguously_classifiable` + +| `safety` +| `arm_auto` only for the safe classes; `flag` for everything else + +| `pool` +| `P0`–`P3`; poison dispositions run at `P1`, routine bumps at `P2` + +| `pin_delta` +| one row per changed pin: `action`, `from`, `to`, `source` + (`tags` \| `body-claim`) +|=== + +Two rules of evidence keep the version numbers honest: + +. The diff's `uses:` refs are the ground truth for *what changed*. +. Upstream tags resolve a ref to a version (`source: tags`); a dependabot body + claim fills a gap only when the tags cannot (`source: body-claim`); if the + two disagree the decision is `version_claims_conflict`, never a guess. + +A lockfile-only PR has no `uses:` line to resolve, so it needs a body claim to +be classifiable at all. `unresolved` without one. + +== The four sweeps + +All four are dry runs by default; none of them writes without `--execute` or +`--rewrite`. + +=== `estate-pin-integrity.sh` + +Find every denylisted pin in an estate; with `--rewrite`, produce the +substituted file trees ready for a branch and a PR. + +* Flags: `--org` (repeatable), `--policy`, `--out DIR`, `--via search|clone`, + `--rewrite`, `--max-repos N`. +* Artifacts under `--out`: `pin-findings.jsonl`, `pin-repair-plan.jsonl`, + `patched//` trees; one squash PR is planned per repository + (`hypatia/pin-integrity-known-good`). +* Exit: `0` clean, `1` error, `2` findings, `3` nothing to do. +* A repair that cannot be proved is a finding, not an action: the substitution + refuses when the policy carries no `known_good_sha`, and it refuses when the + rewrite would drop a `uses:` site. + +=== `estate-pr-automerge.sh` + +Classify every open PR in an estate and (with `--execute`) merge the armed +ones, close the poisoned ones, and delete branches. + +* Flags: `--org` (repeatable), `--policy`, `--out DIR`, `--cache DIR`, + `--max-repos N`, `--from FILE`, `--execute`. +* `--execute` re-checks, per decision, that the PR's head SHA is unchanged and + that the live diff still contains no denylisted ref. A stale or refused + decision is skipped and logged, never applied blind. Every merge and close + deletes the branch. +* Archived repositories are enumerated, not skipped: they get + `close_archived_repo` and never a merge. + +=== `estate-absence-intake.sh` + +Turn unresolvable absences into issues exactly once. + +* Flags: `--org` (repeatable), `--policy`, `--out DIR`, `--max-new N`, + `--execute`. +* Dedupe key: `` in the issue body. A second + run comments, never re-files; a closed issue is never reopened. +* A class over the roll-up threshold becomes one estate-wide listing issue + instead of N repository issues. +* Exit: `2` dry run, `0` executed. + +=== `estate-stats.sh` + +Emit the single estate-wide statistics artifact the private dashboard reads. + +* Flags: `--org` (repeatable), `--out FILE`, `--decisions FILE`, + `--scans DIR`, `--max-issues N`. +* Output: `estate-stats.json` plus a human-readable `.adoc` companion. +* Every section carries a `provenance` block; a metric that could not be + measured is reported as `unavailable`, never as `0`. Numbers that were never + fetched must not be able to look green. + +== The dashboard contract + +Producer: `estate-stats.sh` → `docs/status/estate-stats.json` +(schema version from `stats_output.schema_version`). + +Consumer: the private farm, at +`hyperpolymath/.git-private-farm` → `metadatastician/berrywiki/data/estate-stats.json`, +with dated copies under the history directory recorded in `stats_output`. + +Sections: `repositories` (archived/empty-description/topics-low counts), +`pull_requests` (counts by disposition, poisoned count, oldest/median age), +`issues` (open count, distinct causes, never-commented), `signals` +(test-coverage empties, failing tests, bench ratios, where measured). +Thresholds come from `stats_thresholds`; the artifact does not invent its own. + +== Operating procedure + +. `./scripts/sweeps/estate-pin-integrity.sh --org hyperpolymath --out /tmp/pin` + — read the plan; `--rewrite` to produce patched trees. +. `./scripts/sweeps/estate-pr-automerge.sh --org hyperpolymath --org metadatastician --out /tmp/pr` + — read `pr-decisions.jsonl` and the summary. +. Review the `flag` rows by hand. They are the cases the machinery refuses to + decide: majors, conflicts, unresolvable deltas, licence touches, archived + repositories, human authors. +. `--execute` on a host that owns a write token. +. `./scripts/sweeps/estate-stats.sh …` after the run; the artifact is the + record of what changed. + +== What this machinery deliberately will not do + +* Merge a major bump, a security advisory, a workflow add/remove, a + `permissions:`/`on:`/`if:`/`env:`/`secrets:` delta, or a human's PR. +* Merge a licence/SPDX touch. A licence is a claim about what the project + *is*; it routes to the owner. +* Reopen a closed absence issue. +* Trust a PR title, a PR body, or an inline comment as the source of truth for + what a pin points at. The ref after `@` is the claim; everything else is + annotation. +* Report an unmeasured metric as zero. + +== Tests + +`test/rules/pr_automerge_test.exs` and `test/rules/pin_integrity_test.exs` fix +the behaviour of both rule modules, including the cases that matter most: +the poisoned commit under a correct-looking comment (`PI002`), the rollback +that relabels without re-pinning, the diff-marker/YAML-dash line shape, +string-keyed JSON records, the licence guard, and the `MGX-001` proof. The +shell engine's `relabel_comment` is kept byte-for-byte identical to the +Elixir `relabel/2`; where they diverge, the rename-without-repin bug comes +back. diff --git a/lib/rules/pin_integrity.ex b/lib/rules/pin_integrity.ex new file mode 100644 index 00000000..6919d6dc --- /dev/null +++ b/lib/rules/pin_integrity.ex @@ -0,0 +1,694 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.Rules.PinIntegrity do + @moduledoc """ + Pin-level integrity rules for workflow and lockfile content. + + Rule IDs PI001-PI005. Every function here is **pure**: it takes file + content and the policy map, and returns findings or a rewritten string. + No network, no repo handle, no clock. That is deliberate — this is the + module that a token-bearing actuator re-runs to *independently re-prove* + a decision the brain made, so it must be drivable from either side. + + ## Why these rules exist + + The estate lost a day to `github/codeql-action` **v4.38.1** + (`1c5b675653bb5c22dbe9b12b556ec555138e09fd`). GitHub rejects that commit + at workflow start-up — `startup_failure`, zero jobs, no logs — so CodeQL, + Hypatia Security Scan and Scorecard died on every repo it reached. + + The estate rolled it back. Then it came back, twice, because the + rollback **relabelled the pin without changing it**: files across the + estate now carry + + uses: github/codeql-action/init@1c5b6756... # v4.38.0 + + which is the 4.38.1 commit wearing a 4.38.0 label. A check that greps + for the version string sees a correct pin. A check that greps for the + commit sees 108 files across 84 repositories. + + So: **pin identity is the SHA, never the comment.** PI002 exists + specifically to catch the label/pin divergence, and it is the rule that + would have stopped the second round. + + ## Rule catalogue + + | Rule | Severity | Fires when | + |---|---|---| + | PI001 | critical | a `uses:` ref resolves to a denylisted SHA or version | + | PI002 | high | the inline comment names a version the ref does not resolve to | + | PI003 | high | a moving ref (`stable`, `latest`, ...) carries a SHA pin | + | PI004 | medium | a lockfile entry disagrees with the workflow it locks | + | PI005 | info | a pin whose upstream ref has moved (stale-pin, advisory) | + + PI001 and PI002 are the pair that matters: PI001 alone is defeated by a + mislabelled pin, and PI002 alone is defeated by a confidently wrong + comment on a *good* pin. Their conjunction is the invariant. + + ## Dispatch + + Findings route to `Patch-Bridge` as `pin_rollback_denylisted` + (auto-executable, because the repair is a literal token substitution) + or to the owner as `flag` when the repair is not a substitution. + """ + + @uses_regex ~r/^\s*-?\s*uses:\s*(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_./-]*?)@(?[^\s#]+)\s*(?:#\s*(?.*?))?\s*$/ + + @doc """ + Parse every `uses:` pin site in a workflow file. + + Returns a list of maps with `:line`, `:action` (the bare action path as + written, e.g. `github/codeql-action/init`), `:action_base` (the first + two path segments, which is what an allow/deny list keys on), + `:ref`, and `:comment`. + """ + @spec pin_sites(String.t()) :: [map()] + def pin_sites(content) when is_binary(content) do + content + |> String.split("\n") + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + case Regex.named_captures(@uses_regex, line) do + %{"action" => action, "ref" => ref} = caps -> + [ + %{ + line: number, + action: action, + action_base: action_base(action), + ref: ref, + comment: Map.get(caps, "comment", "") || "" + } + ] + + _ -> + [] + end + end) + end + + def pin_sites(_), do: [] + + @doc """ + The first two segments of an action path. + + `github/codeql-action/init` → `github/codeql-action`, because the + denylist names the *release unit*, not the sub-action. Denying + `init` but allowing `analyze` would leave half a CodeQL install + poisoned. + """ + @spec action_base(String.t()) :: String.t() + def action_base(action) do + case String.split(action, "/") do + [owner, name | _rest] -> "#{owner}/#{name}" + _ -> action + end + end + + @doc """ + Does this ref sit on the policy denylist? + + Accepts a SHA (`1c5b6756...`), a tag (`v4.38.1`), or a version + (`4.38.1`) — the estate uses all three forms and the decision must not + depend on which spelling a repo happens to carry. + """ + @spec denylisted?(map(), String.t(), String.t()) :: nil | map() + def denylisted?(policy, action, ref) do + base = action_base(action) + normalised = normalise_ref(ref) + + policy + |> Map.get("pin_denylist", []) + |> Enum.find(fn entry -> + Map.get(entry, "action") == base and + (normalised in normalise_all(Map.get(entry, "blocked_shas", [])) or + normalised in normalise_all(Map.get(entry, "blocked_versions", [])) or + normalised in normalise_all(Map.get(entry, "blocked_refs", []))) + end) + end + + # Both sides of the comparison get the same treatment: the policy writes + # `v4.38.1` while a workflow may write `4.38.1`, and a denylist that only + # matches one spelling is a denylist with a hole in it. + defp normalise_all(list), do: Enum.map(list, &normalise_ref/1) + + defp normalise_ref(ref) do + ref + |> String.trim() + |> String.trim_leading("v") + |> case do + # `v4.38.1` and `4.38.1` are the same claim about the same artefact. + other -> other + end + end + + @doc """ + The known-good replacement for a denylisted pin, or `nil`. + """ + @spec known_good(map(), String.t()) :: nil | map() + def known_good(policy, action) do + base = action_base(action) + + case Enum.find(Map.get(policy, "pin_denylist", []), &(Map.get(&1, "action") == base)) do + %{"known_good_sha" => sha} = entry when is_binary(sha) -> + %{sha: sha, version: Map.get(entry, "known_good_version"), rule: Map.get(entry, "id")} + + _ -> + nil + end + end + + # ─── PI001 · denylisted pin ──────────────────────────────────────────── + + @doc """ + PI001: a `uses:` ref resolves to a denylisted SHA or version. + + Severity `:critical` for every entry carrying `"severity": "critical"` + in the policy (currently PIN-001); `:high` otherwise. + """ + @spec pi001_denylisted_pin(String.t(), map(), keyword()) :: [map()] + def pi001_denylisted_pin(content, policy, opts \\ []) do + path = Keyword.get(opts, :path, "") + repo = Keyword.get(opts, :repo, "") + + content + |> pin_sites() + |> Enum.flat_map(fn site -> + case denylisted?(policy, site.action, site.ref) do + nil -> + [] + + entry -> + [ + finding( + "PI001", + severity_atom(entry), + repo, + path, + site.line, + """ + `#{site.action}@#{site.ref}` is on the pin denylist (#{entry["id"]}). + + #{entry["reason"]} + + Replacement: #{format_replacement(entry)} + """, + %{ + action: site.action, + ref: site.ref, + denylist_id: entry["id"], + known_good_sha: entry["known_good_sha"], + repair: "substitution" + } + ) + ] + end + end) + end + + # ─── PI002 · mislabelled pin ─────────────────────────────────────────── + + @doc """ + PI002: the inline comment names a version the ref does not resolve to. + + This is the rule the estate needed on 2026-09-23 and did not have. + A file carrying + + uses: github/codeql-action/init@1c5b6756... # v4.38.0 + + is *poisoned and labelled correct*. Only a rule that compares the + comment against the resolved ref can see it. + + `resolution` is a map of ref → version, or ref → `%{version: v, tainted: bool}`. + When the ref is absent from the map the rule cannot judge and stays + silent — an unresolved claim is not a finding. + """ + @spec pi002_mislabelled_pin(String.t(), map(), keyword()) :: [map()] + def pi002_mislabelled_pin(content, resolution, opts \\ []) do + path = Keyword.get(opts, :path, "") + repo = Keyword.get(opts, :repo, "") + + content + |> pin_sites() + |> Enum.flat_map(fn site -> + with claimed when is_binary(claimed) <- claimed_version(site.comment), + true <- claimed != "", + resolved when is_binary(resolved) <- resolve_version(resolution, site.ref), + true <- not version_prefix?(normalise_ref(claimed), normalise_ref(resolved)) do + [ + finding( + "PI002", + :high, + repo, + path, + site.line, + """ + Pin comment claims `#{claimed}` but `#{site.ref}` resolves to `#{resolved}`. + + The comment is what a human reviewer reads and what a + version-string grep matches, so a divergence here makes a + bad pin look like a good one. This is the pattern that let + the codeql-action 4.38.1 rollback fail silently across the + estate. + """, + %{ + action: site.action, + ref: site.ref, + claimed: claimed, + resolved: resolved, + repair: "relabel_or_resubstitute" + } + ) + ] + else + _ -> [] + end + end) + end + + @doc """ + Pull a version out of a pin comment. + + Handles the estate's actual comment shapes: + + # v4.38.0 + # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + # 4.38.0 + # v3 + # Pinned to v1.2.3 — do not move + + Returns `nil` when the comment makes no version claim. + """ + @spec claimed_version(String.t()) :: nil | String.t() + def claimed_version(comment) when is_binary(comment) do + # `v3` is a real shape in the estate (dictask carries a poisoned pin + # annotated `# v3`), so a bare major behind a `v` counts. A bare number + # without the `v` does not: `# 2 jobs` is prose, not a version claim. + case Regex.run(~r/\b(?:v(\d+(?:\.\d+)*)|(\d+\.\d+(?:\.\d+)?))\b/, comment) do + [_, version, _] when version != "" -> version + [_, _, version] when version != "" -> version + _ -> nil + end + end + + def claimed_version(_), do: nil + + # A claim that names only the leading segments (`v3` against `3.1.0`) is + # coarse, not wrong. Only a claim that contradicts a segment it does name + # is a mislabel. + defp version_prefix?(claimed, resolved) do + claimed_parts = String.split(claimed, ".") + resolved_parts = String.split(resolved, ".") + + length(claimed_parts) <= length(resolved_parts) and + Enum.take(resolved_parts, length(claimed_parts)) == claimed_parts + end + + defp resolve_version(resolution, ref) do + case Map.get(resolution, ref) do + %{version: v} -> v + v when is_binary(v) -> v + _ -> nil + end + end + + # ─── PI003 · locked moving ref ──────────────────────────────────────── + + @doc """ + PI003: a moving ref carries a SHA pin. + + `dtolnay/rust-toolchain@stable` re-points on every Rust release. A + lockfile that pins the SHA it happened to have kills every consumer at + `Set up job` from the next release onward (hypatia CI-1: 4 workflows, + 13 jobs, all four open dependabot PRs). + + The cure is structural — do not lock moving refs — so the finding + carries `repair: "unpin_moving_ref"`, never `"substitution"`. + """ + @spec pi003_locked_moving_ref(String.t(), map(), keyword()) :: [map()] + def pi003_locked_moving_ref(content, policy, opts \\ []) do + path = Keyword.get(opts, :path, "") + repo = Keyword.get(opts, :repo, "") + moving = Map.get(policy, "moving_refs_never_lockable", []) + locked = locked_refs(content) + + for {ref, {action, line}} <- locked, + ref in moving do + finding( + "PI003", + :high, + repo, + path, + line, + """ + `#{action}@#{ref}` is a moving ref locked to a fixed SHA. + + Moving refs cannot be lockfile-pinned: the pin stops matching the + ref at the next upstream release and every job using it dies before + checkout. Remove the lockfile entry rather than re-pinning it. + """, + %{action: action, ref: ref, repair: "unpin_moving_ref"} + ) + end + end + + @doc """ + Extract `action@sha` pairs from a `gh actions-lock` file (TOML-ish), + keyed by the resolved SHA or ref. Used by PI003 and PI004. + """ + @spec locked_refs(String.t()) :: %{String.t() => {String.t(), integer()}} + def locked_refs(content) do + content + |> String.split("\n") + |> Enum.with_index(1) + |> Enum.flat_map(fn {line, number} -> + case Regex.run(~r/'(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_./-]+)@(?[^\s']+)'/, line) do + nil -> + [] + + captures -> + action = Enum.at(captures, 1) + ref = Enum.at(captures, 2) + [{ref, {action_base(action), number}}] + end + end) + |> Map.new() + end + + # ─── PI004 · lock/spec divergence ───────────────────────────────────── + + @doc """ + PI004: a workflow pins a SHA the lockfile disagrees with. + + hypatia on 2026-09-26: `actions.lock` pinned + `github/codeql-action@b96794f0...` while `codeql.yml` and + `security-policy.yml` carried `1c5b6756...`. The lock was right and the + workflows were wrong, so any gate that verifies the lockfile rather + than the workflows passes a poisoned repo. The direction of the + divergence does not matter — agreement does. + """ + @spec pi004_lock_divergence(String.t(), String.t(), keyword()) :: [map()] + def pi004_lock_divergence(workflow_content, lock_content, opts \\ []) do + path = Keyword.get(opts, :path, "") + repo = Keyword.get(opts, :repo, "") + locked = locked_refs(lock_content) + + workflow_content + |> pin_sites() + |> Enum.flat_map(fn site -> + locked_refs_for_action = + locked + |> Enum.filter(fn {_ref, {action, _line}} -> action == site.action_base end) + |> Enum.map(fn {ref, _} -> ref end) + + cond do + locked_refs_for_action == [] -> + [] + + site.ref in locked_refs_for_action -> + [] + + true -> + [ + finding( + "PI004", + :medium, + repo, + path, + site.line, + """ + `#{site.action}` is pinned to `#{site.ref}` here, but the lockfile + pins `#{Enum.join(locked_refs_for_action, ", ")}`. + + The lockfile is the thing a verify step reads, so a divergence + makes a poisoned workflow look compliant. + """, + %{ + action: site.action, + workflow_ref: site.ref, + lock_refs: locked_refs_for_action, + repair: "reconcile_lock_and_workflow" + } + ) + ] + end + end) + end + + # ─── Mechanical repair ──────────────────────────────────────────────── + + @doc """ + Rewrite every denylisted pin site in `content` to its known-good SHA. + + This is the whole "it is really just a file merge problem" case: the + repair is a token substitution that either applies exactly or does not + apply at all. Returns + `{:ok, new_content, excisions}` | `{:error, reason}`. + + Refuses — rather than guesses — when: + + * a denylisted site has no `known_good_sha` in the policy; + * the substituted content would still parse fewer pin sites than it + started with (a rewrite that deletes a `uses:` line is not a + substitution). + + The refusal path is the point. A repair that cannot be proved is a + finding, not an action. + """ + @spec substitute_denylisted_pins(String.t(), map(), keyword()) :: + {:ok, String.t(), [map()]} | {:error, String.t()} + def substitute_denylisted_pins(content, policy, _opts \\ []) do + lines = String.split(content, "\n") + sites = pin_sites(content) + hits = Enum.filter(sites, &denylisted?(policy, &1.action, &1.ref)) + + cond do + hits == [] -> + {:ok, content, []} + + Enum.any?(hits, &(known_good(policy, &1.action) == nil)) -> + {:error, "no known_good_sha for a denylisted action in this file"} + + true -> + {new_content, excisions} = + Enum.reduce(hits, {content, []}, fn site, {acc, log} -> + original = Enum.at(lines, site.line - 1) || "" + %{sha: sha, version: version} = known_good(policy, site.action) + + rewritten = + original + |> String.replace("@" <> site.ref, "@" <> sha) + |> relabel_line(version) + + excision = %{ + line: site.line, + action: site.action, + from_ref: site.ref, + to_ref: sha, + to_version: version, + comment_before: site.comment, + comment_after: comment_of(rewritten) + } + + # Whole-line replacement, so two identical pin lines are treated + # identically and neither is touched twice. + {String.replace(acc, original, rewritten), log ++ [excision]} + end) + + if length(pin_sites(new_content)) == length(sites) do + {:ok, new_content, excisions} + else + {:error, "substitution would remove a pin site — refusing"} + end + end + end + + defp comment_of(line) do + case String.split(line, "#", parts: 2) do + [_head, comment] -> comment + _ -> "" + end + end + + defp relabel_line(line, version) when is_binary(version) do + case String.split(line, "#", parts: 2) do + [head, comment] -> head <> "#" <> relabel(comment, version) + _ -> line + end + end + + defp relabel_line(line, _version), do: line + + @doc """ + Relabel a pin's inline comment — but only when the comment **leads** with a + version claim. + + Both estate shapes are covered: + + "# v3" -> "# v4.38.0" + "# v4.38.0 (4.38.1 blocked estate-wide; …)" -> unchanged + + and prose that merely mentions a version mid-sentence is returned + byte-identical, because mangling a sentence to fix a label trades a silent + wrong pin for a silent wrong sentence. + + Must stay behaviourally identical to `relabel_comment` in + `scripts/sweeps/estate-pin-integrity.sh`. Two readers, one policy, one edit. + """ + @spec relabel(String.t(), nil | String.t()) :: String.t() + def relabel(comment, version) when is_binary(comment) and is_binary(version) do + body = String.replace_prefix(comment, "#", "") + + case Regex.run(~r/^\s*/, body) do + [lead] -> + trimmed = String.slice(body, String.length(lead)..-1//1) + + case Regex.run(~r/^(v?\d+(?:\.\d+)*)(?:\s|$)/, trimmed) do + [_whole, claim] -> "#" <> lead <> String.replace_prefix(trimmed, claim, "v" <> version) + _ -> comment + end + + _ -> + comment + end + end + + def relabel(comment, _version), do: comment + + @doc """ + Prove the meta-guard exemption `MGX-001` (`pin_only_workflow_edit`). + + Given the removed and added lines of a diff, answer only one question: + **is every changed line a `uses:` pin token substitution on a line that + exists on both sides, with the action name unchanged?** + + Everything else — a new step, a changed `permissions:` block, a moved + `if:`, an added `on:` trigger — makes this `false`, which forces the + decision back to `change_level = meta` and `safety = flag`. + + The actuator calls this on the raw diff. It never accepts the brain's + claim that the edit was pin-only; it re-derives it. + """ + @spec pin_only_edit?([String.t()], [String.t()], map()) :: boolean() + def pin_only_edit?(removed, added, policy) do + removed_sites = lines_to_sites(removed) + added_sites = lines_to_sites(added) + + pinned_removed = removed_sites |> Enum.map(& &1.ref) |> Enum.sort() + pinned_added = added_sites |> Enum.map(& &1.ref) |> Enum.sort() + + cond do + removed == [] or added == [] -> + false + + length(removed_sites) != length(removed) or length(added_sites) != length(added) -> + # At least one changed line is not a `uses:` line. + false + + Enum.sort(Enum.map(removed_sites, & &1.action)) != + Enum.sort(Enum.map(added_sites, & &1.action)) -> + # An action was swapped for a different action. + false + + Enum.any?(added_sites, &denylisted?(policy, &1.action, &1.ref)) -> + # The substitution moves *onto* the denylist. + false + + pinned_removed == pinned_added -> + # Nothing actually changed. + false + + true -> + true + end + end + + defp lines_to_sites(lines) do + lines + |> Enum.map(&Regex.named_captures(@uses_regex, &1)) + |> Enum.flat_map(fn + %{"action" => action, "ref" => ref} = caps -> + [%{action: action, action_base: action_base(action), ref: ref, comment: Map.get(caps, "comment", "") || ""}] + + _ -> + [] + end) + end + + # ─── PI005 · stale pin (advisory) ───────────────────────────────────── + + @doc """ + PI005: a pin whose upstream ref has moved on, reported as information. + + Deliberately `:info` and deliberately last. A stale pin is not a + defect; it becomes one only when it is behind a security fix. The + estate's problem in September 2026 was never that pins were *old* — it + was that a fresh pin was *poisoned* and nobody could tell the two apart + because both arrived as `chore(deps): bump …`. + """ + @spec pi005_stale_pin(String.t(), map(), keyword()) :: [map()] + def pi005_stale_pin(content, latest_versions, opts \\ []) do + path = Keyword.get(opts, :path, "") + repo = Keyword.get(opts, :repo, "") + + content + |> pin_sites() + |> Enum.flat_map(fn site -> + case Map.get(latest_versions, site.action_base) do + latest when is_binary(latest) -> + if normalise_ref(latest) != normalise_ref(site.ref) and + claimed_version(site.comment) not in [nil, latest] do + [ + finding( + "PI005", + :info, + repo, + path, + site.line, + "`#{site.action}` pins #{site.ref}; upstream is at #{latest}.", + %{action: site.action, ref: site.ref, latest: latest, repair: "advisory"} + ) + ] + else + [] + end + + _ -> + [] + end + end) + end + + # ─── Shared finding shape ───────────────────────────────────────────── + + defp finding(rule_id, severity, repo, path, line, description, detail) do + %{ + rule_id: rule_id, + severity: severity, + repo: repo, + path: path, + line: line, + description: String.trim(description), + detail: detail + } + end + + defp severity_atom(%{"severity" => "critical"}), do: :critical + defp severity_atom(%{"severity" => "high"}), do: :high + defp severity_atom(%{"severity" => "medium"}), do: :medium + defp severity_atom(_), do: :high + + defp format_replacement(%{"known_good_sha" => sha, "known_good_version" => version}) + when is_binary(sha) do + suffix = + case version do + v when is_binary(v) -> " (v" <> v <> ")" + _ -> "" + end + + "pin to `" <> sha <> "`" <> suffix + end + + defp format_replacement(%{"replacement" => replacement}), do: replacement + defp format_replacement(_), do: "manual review" +end diff --git a/lib/rules/pr_automerge.ex b/lib/rules/pr_automerge.ex new file mode 100644 index 00000000..780f12d8 --- /dev/null +++ b/lib/rules/pr_automerge.ex @@ -0,0 +1,517 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.Rules.PrAutomerge do + @moduledoc """ + Decide whether a pull request is an unambiguous bump or chore that may be + merged and its branch deleted, and if not, exactly why not. + + Rule IDs PA001-PA006. Every function is pure: a PR record and the policy map + in, a decision out. No network. The `gh`-driven plumbing lives in + `Mix.Tasks.Hypatia.Automerge` and in `scripts/sweeps/estate-pr-automerge.sh`; + the shell sweep (`scripts/sweeps/estate-pr-automerge.sh`) is also the + token-bearing actuator, and it re-derives these rules against the live diff + before it acts. Two readers, one policy file. + + ## Why this is not a title matcher + + On 2026-09-26 the estate had 32 open PRs, every one a `chore(deps): bump …` + from dependabot. Every one of them looked safe. 25 of them re-introduced + `github/codeql-action` v4.38.1 — the commit GitHub rejects at workflow + start-up — and at least two hid a *major* bump (`actions/checkout` + v4.1.7 → v7.0.1) behind a title that says only "bump the actions group". + + A title matcher merges 32 of them. The estate did merge some of them: + nexia-list#107 and dictask#65 landed poisoned pins and both repositories are + red on main as a result. + + ## The order of the checks is the safety property + + PA001 (denylisted pin) is checked first, before anything that could grant + permission, because every later check answers "is this a routine change?" + and the pin question answers "is this change *safe*?" — different questions, + and the second one dominates. A grouped update carrying a wanted minor bump + *and* the poisoned pin is not "mostly fine"; it is a PR whose net effect on + main is a broken workflow. + + ## Rule catalogue + + | Rule | Severity | Fires when | + |---|---|---| + | PA001 | critical | an added line puts a denylisted pin back in the tree | + | PA002 | high | a version delta crosses a major boundary | + | PA003 | high | the diff and the PR body disagree about the versions | + | PA004 | medium | the version delta cannot be resolved at all | + | PA005 | medium | a dependency-manifest change with no lockfile change | + | PA006 | low | the PR is not automatable for a structural reason | + + ## Dispositions + + * `:auto_merge` — merge, delete the branch + * `:excise_poison_then_merge` — the poisoned hunk is removed, the rest + merged, the branch deleted + * `:close_poison_only` — nothing here is wanted; close and delete + * `:close_poison_and_majors` — poisoned *and* carrying majors + * `:close_archived_repo` — the repository is archived; it can never merge + * `:flag` — a human decides + + `:excise_poison_then_merge` is the case the phrase "it is really just a file + merge problem" describes: the repair is a token substitution inside one file, + so the wanted half of a grouped update can be kept without waiting for + dependabot to re-raise it. + """ + + @doc """ + Classify a PR record against the policy. + + `pr` is a map with: + + * `:author` — login of the PR author + * `:repo_archived` — whether the repository is archived (default `false`) + * `:files` — list of `%{filename: String.t(), patch: String.t()}` + * `:body` — the PR body, used only as a corroborating claim source + * `:version_resolution` — map of `{action, ref} => version | nil`, i.e. + what the upstream tags say a ref is. Absent entries are *unknown*, never + assumed. + + Returns the decision as a plain map; `decision_manifest/2` turns it into the + frozen manifest shape. + """ + @spec classify(map(), map()) :: map() + def classify(pr, policy) do + files = field(pr, :files, []) + body = field(pr, :body, "") + resolution = field(pr, :version_resolution, %{}) + + claims = body_claims(body) + deltas = pin_deltas(files, resolution, claims) + + poison = poison_sites(files, policy) + lock_only = files != [] and Enum.all?(files, &lockfile?(filename_of(&1), policy)) + meta_only = files != [] and Enum.all?(files, &metadata_path?(filename_of(&1))) + pin_only = files != [] and Enum.all?(files, &pin_lines_only?/1) + manifest_file? = Enum.any?(files, &dependency_manifest?(filename_of(&1))) + licence_touch? = Enum.any?(files, &licence_path?(filename_of(&1))) + + %{ + deltas: deltas, + poison_sites: poison, + major_delta: Enum.any?(deltas, &major_delta?/1), + unresolved: Enum.count(deltas, &(&1.status == :unresolved)), + conflicts: Enum.count(deltas, &(&1.status == :conflict)), + lock_only: lock_only, + meta_only: meta_only, + pin_only: pin_only, + manifest_file: manifest_file?, + licence_touch: licence_touch? + } + |> verdict(pr, policy) + end + + # ─── PA001 · denylisted pin ─────────────────────────────────────────── + + @doc """ + Every added line that puts a denylisted pin site into the tree. + + Keyed on the ref, never on the inline comment: the estate contains files + where the poisoned commit is labelled `# v4.38.0`, and a version-string + check sees nothing wrong with them. + """ + @spec poison_sites([map()], map()) :: [map()] + def poison_sites(files, policy) do + policy + |> Map.get("pin_denylist", []) + |> Enum.flat_map(fn entry -> + files + |> Enum.flat_map(fn file -> + file + |> added_lines() + |> Enum.flat_map(fn line -> + case parse_pin(line) do + %{action: action, ref: ref} -> + if action_base(action) == Map.get(entry, "action") and + denylisted_ref?(entry, ref) do + [ + %{ + file: filename_of(file), + line: String.trim(line), + denylist_id: Map.get(entry, "id"), + severity: Map.get(entry, "severity", "critical") + } + ] + else + [] + end + + _ -> + [] + end + end) + end) + end) + end + + # The ref that sits directly after `@` is the claim; a version string + # anywhere else on the line is an annotation. The estate's rollback comment + # reads `# v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)` — matching + # on the whole line would call that file poisoned, which is the same class + # of mistake as trusting the comment in the first place. + defp denylisted_ref?(entry, ref) do + blocked = + Map.get(entry, "blocked_shas", []) ++ + Map.get(entry, "blocked_versions", []) ++ Map.get(entry, "blocked_refs", []) + + normalised = normalise_ref(ref) + Enum.any?(blocked, &(normalise_ref(&1) == normalised)) + end + + defp normalise_ref(ref) when is_binary(ref) do + ref |> String.trim() |> String.replace_prefix("v", "") + end + + defp normalise_ref(_), do: "" + + @doc """ + Is every changed line in this file a `uses:` pin line? + + This is the proof obligation behind the meta-guard exemption `MGX-001`. A + workflow edit that is *not* pin-only — a new step, a changed `permissions:` + block, a moved `if:` — fails this, which is what keeps the reflexivity guard + from being talked around by a plausible title. + """ + @spec pin_lines_only?(map()) :: boolean() + def pin_lines_only?(file) do + lines = content_lines(patch_of(file)) + + lines != [] and Enum.all?(lines, fn line -> pin_line?(line) end) + end + + # ─── PA002/PA003/PA004 · version deltas ─────────────────────────────── + + @doc """ + Version deltas for every action this PR re-pins. + + `status` is one of `:ok`, `:unresolved` (no source could place a version on + the refs) or `:conflict` (upstream tags and the PR body disagree). `source` + records which source produced the versions that were used, so a reviewer can + see exactly what the decision rested on. + """ + @spec pin_deltas([map()], map(), [map()]) :: [map()] + def pin_deltas(files, resolution, claims) do + files + |> Enum.flat_map(fn file -> + removed = file |> removed_lines() |> Enum.filter(&pin_line?/1) + added = file |> added_lines() |> Enum.filter(&pin_line?/1) + + removed + |> Enum.flat_map(fn line -> + old = parse_pin(line) + + case Enum.find(added, &(parse_pin(&1).action == old.action)) do + nil -> + [] + + new_line -> + new = parse_pin(new_line) + + if new.ref == old.ref do + [] + else + from = resolve(resolution, old.action, old.ref) + to = resolve(resolution, new.action, new.ref) + claim = claim_for(claims, old.action) + + cond do + from != nil and to != nil and claim != nil and + (claim.from != from or claim.to != to) -> + delta(old, new, from, to, :conflict, "tags") + + from != nil and to != nil -> + delta(old, new, from, to, :ok, "tags") + + claim != nil -> + delta(old, new, claim.from, claim.to, :ok, "body-claim") + + true -> + delta(old, new, nil, nil, :unresolved, "none") + end + |> Map.put(:file, filename_of(file)) + end + end + end) + end) + end + + defp delta(old, new, from, to, status, source) do + %{ + action: old.action, + from: from, + to: to, + status: status, + source: source, + file: nil, + major?: from != nil and to != nil and major_of(from) != major_of(to) + } + end + + @doc "Is this delta a major-version crossing? Unknown versions are never a yes." + @spec major_delta?(map()) :: boolean() + def major_delta?(%{major?: flag}), do: flag == true + def major_delta?(_), do: false + + @doc """ + The claim list dependabot writes into the PR body. + + Both shapes are captured — `Updates \\`owner/action\\` from A to B` (grouped) + and `Bumps [name](url) from A to B.` (single) — because the estate uses + both and a claim parser that understands one of them silently converts half + the estate into "unverifiable". + """ + @spec body_claims(String.t()) :: [map()] + def body_claims(body) when is_binary(body) do + ~r/(?:Updates|Bumps)\s+\[?`?([A-Za-z0-9_.-]+(?:\/[A-Za-z0-9_.-]+)*)`?\]?(?:\([^)]*\))?\s+from\s+([0-9][^\s]*)\s+to\s+([0-9][^\s]*)/ + + |> Regex.scan(body) + |> Enum.map(fn [_, action, from, to] -> + # Dependabot ends the sentence with a full stop; the version does not + # have one. `to [0-9][^ .]*` (an earlier shape) silently turned + # `4.38.1` into `4` — which is how a major bump can hide in a minor. + %{action: action, from: trim_dot(from), to: trim_dot(to)} + end) + end + + def body_claims(_), do: [] + + defp trim_dot(version), do: String.trim_trailing(version, ".") + + # ─── Verdict ───────────────────────────────────────────────────────── + + defp verdict(scan, pr, policy) do + author = field(pr, :author) + repo_archived = field(pr, :repo_archived) == true + + base = %{ + change_class: "bump", + change_level: if(scan.pin_only, do: "object", else: "meta"), + route: "Patch-Bridge", + method: "squash", + pool: "P2", + safety: "flag", + attestations: [ + %{bot: "hypatia", verdict: "approve", confidence: 0.9, rationale: "classified from the diff"} + ] + } + + cond do + repo_archived -> + reject(base, :close_archived_repo, "repository_is_archived_cannot_merge", "P3") + + not dependency_bot?(author, policy) -> + reject(base, :flag, "not_a_dependency_bot", "P3") + + scan.poison_sites != [] and scan.major_delta -> + reject(base, :close_poison_and_majors, "introduces_denylisted_pin_and_major_bumps", "P1") + + scan.poison_sites != [] and scan.pin_only -> + accept(base, :close_poison_only, "introduces_denylisted_pin", "P1") + + scan.poison_sites != [] -> + accept(base, :excise_poison_then_merge, "introduces_denylisted_pin_alongside_wanted_updates", "P1") + + scan.major_delta -> + reject(base, :flag, "major_version_delta", "P2") + + scan.conflicts > 0 -> + reject(base, :flag, "version_claims_conflict", "P2") + + scan.unresolved > 0 -> + reject(base, :flag, "pin_delta_unresolvable", "P2") + + scan.licence_touch -> + # NA-005-adjacent by intent: a licence/SPDX touch is a claim about + # what the project *is*, so it routes to the owner, never the robot. + reject(base, :flag, "touches_licence_requires_owner_review", "P2") + + scan.lock_only -> + accept(base, :auto_merge, "awaiting_required_checks", "P2") + + scan.pin_only -> + accept(base, :auto_merge, "awaiting_required_checks", "P2") + + scan.meta_only -> + accept(base, :auto_merge, "awaiting_required_checks", "P3") + + scan.manifest_file -> + reject(base, :flag, "dependency_manifest_not_lockfile", "P2") + + true -> + reject(base, :flag, "not_unambiguously_classifiable", "P2") + end + end + + defp accept(base, disposition, blocked_by, pool) do + base + |> Map.merge(%{safety: "arm_auto", pool: pool, disposition: disposition, blocked_by: blocked_by}) + end + + defp reject(base, disposition, blocked_by, pool) do + base + |> Map.merge(%{safety: "flag", pool: pool, disposition: disposition, blocked_by: blocked_by}) + end + + @doc """ + Render a decision as the frozen merge-orchestration manifest. + + Conforms to + `docs/design/merge-orchestration/schemas/decision-manifest.schema.json`; + the two contract invariants hold by construction — any denial sets + `safety: "flag"` and records a veto, and a `meta` change level can only + reach `arm_auto` through the `MGX-001` pin-only exemption, which the + actuator re-proves from the diff. + """ + @spec decision_manifest(map(), map()) :: map() + def decision_manifest(decision, pr) do + vetoes = + if decision.safety == "flag" do + [%{bot: "Patch-Bridge", reason: decision.blocked_by}] ++ + if decision.change_level == "meta", + do: [%{bot: "hypatia", reason: "change_level=meta"}], + else: [] + else + [] + end + + %{ + "pr" => %{ + "repo" => field(pr, :repo, ""), + "number" => field(pr, :number, 0), + "head_sha" => field(pr, :head_sha, ""), + "base" => field(pr, :base, "main"), + "author" => field(pr, :author, ""), + "author_kind" => "dependabot" + }, + "change_class" => decision.change_class, + "change_level" => decision.change_level, + "route" => %{"authority_bot" => decision.route, "contributing_bots" => ["hypatia"]}, + "method" => decision.method, + "method_basis" => "repo-default", + "safety" => decision.safety, + "pool" => decision.pool, + "confidence" => nil, + "attestations" => decision.attestations, + "vetoes" => vetoes, + "clamped_by" => if(vetoes == [], do: nil, else: "veto"), + "rationale" => "PA rules · #{decision.disposition} · blocked_by=#{decision.blocked_by}", + "disposition" => Atom.to_string(decision.disposition), + "blocked_by" => decision.blocked_by, + "pin_delta" => + Enum.map(decision.deltas, fn d -> + %{"action" => d.action, "from" => d.from, "to" => d.to, "source" => d.source} + end), + "denylisted_hits" => length(decision.poison_sites), + "timestamp" => DateTime.utc_now() |> DateTime.to_iso8601() + } + end + + # ─── Primitives ────────────────────────────────────────────────────── + + # Patch lines arrive with a diff marker in front of the YAML dash: + # - - uses: actions/checkout@v4.1.7 + # so the marker is part of the match. (Without this, every real diff looks + # like a non-pin change and every PR falls through to `flag`.) + @pin_re ~r/^[-+]?\s*-?\s*uses:\s*(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_./-]*?)@(?[^\s#]+)/ + + @doc "Parse a `uses:` line into `%{action, base, ref}`; `nil` when it is not one." + def parse_pin(line) do + case Regex.named_captures(@pin_re, line) do + %{"action" => action, "ref" => ref} -> + %{action: action, base: action_base(action), ref: ref} + + _ -> + nil + end + end + + @doc "First two path segments of an action, e.g. `github/codeql-action/init` → `github/codeql-action`." + def action_base(action) do + case String.split(action, "/") do + [owner, name | _] -> "#{owner}/#{name}" + _ -> action + end + end + + defp pin_line?(line), do: parse_pin(line) != nil + + defp content_lines(patch) do + patch + |> String.split("\n") + |> Enum.filter(&Regex.match?(~r/^[-+]/, &1)) + |> Enum.reject(&Regex.match?(~r/^[-+]{3}/, &1)) + end + + defp added_lines(file), do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "+")) + defp removed_lines(file), do: file |> patch_of() |> content_lines() |> Enum.filter(&String.starts_with?(&1, "-")) + + defp patch_of(file), do: Map.get(file, :patch) || Map.get(file, "patch") || "" + + defp filename_of(file), do: Map.get(file, :filename) || Map.get(file, "filename") || "" + + # Records arrive both as Elixir maps (tests) and as Jason-decoded JSON + # (the brain's manifests), and in the second case the keys are strings. + # Reading only one spelling turns every archived repo into an active one. + defp field(map, key, default \\ nil) do + case {Map.get(map, key), Map.get(map, Atom.to_string(key))} do + {nil, nil} -> default + {nil, value} -> value + {value, _} -> value + end + end + + defp resolve(resolution, action, ref) do + Map.get(resolution, {action, ref}) || Map.get(resolution, {action_base(action), ref}) || + Map.get(resolution, ref) + end + + defp claim_for(claims, action) do + short = action |> String.split("/") |> List.last() + + Enum.find(claims, fn c -> c.action == action or c.action == short end) || + Enum.find(claims, fn c -> String.ends_with?(c.action, "/" <> short) end) + end + + defp major_of(version) do + case Regex.run(~r/^v?(\d+)/, version) do + [_, major] -> major + _ -> version + end + end + + defp dependency_bot?(author, policy) do + author in Map.get(policy, "dependency_bots", []) + end + + defp lockfile?(filename, policy) do + Path.basename(filename) in Map.get(policy, "lockfile_names", []) + end + + defp metadata_path?(filename) do + Regex.match?(~r/\.(md|adoc|rst|txt)$/, filename) or + String.starts_with?(filename, ["docs/", "doc/", "LICENSES/", ".github/ISSUE_TEMPLATE/"]) or + Regex.match?(~r/(^|\/)(LICENSE|NOTICE|CODEOWNERS|\.gitattributes|\.editorconfig)/, filename) + end + + defp licence_path?(filename) do + base = Path.basename(filename) + + String.starts_with?(base, ["LICENSE", "LICENCE", "COPYING", "NOTICE"]) or + String.contains?(String.upcase(base), "SPDX") or + String.starts_with?(filename, "LICENSES/") + end + + defp dependency_manifest?(filename) do + Regex.match?( + ~r/(^|\/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|Gemfile|composer\.json)$/, + filename + ) + end +end diff --git a/scripts/sweeps/estate-absence-intake.sh b/scripts/sweeps/estate-absence-intake.sh new file mode 100755 index 00000000..4f99b173 --- /dev/null +++ b/scripts/sweeps/estate-absence-intake.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# estate-absence-intake.sh — turn unresolvable repository absences into issues, +# exactly once each, without turning the tracker into a landfill. +# +# WHY THE CAPS ARE THE FEATURE +# +# The estate has 880 open issues. The complaint was not that findings are +# reported — it is that reporting had no ceiling and no deduplication, so the +# signal drowned. Rules: +# +# * one issue per (repository, absence class), keyed by an in-body marker +# `` — a second run comments, never re-files +# * closed issues are NEVER reopened — a human closing it is a decision +# * at most N new issues per repository per run (policy: intake_limits) +# * a class that exceeds the roll-up threshold becomes ONE estate-wide +# listing issue rather than N repository issues +# * an absence hypatia can fix unambiguously is fixed, not filed; only the +# ones that need a judgement call become issues +# +# DRY RUN by default; --execute files them. +# +# usage: estate-absence-intake.sh [--org ORG]... [--policy FILE] [--out DIR] +# [--execute] [--max-new N] +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +POLICY="$REPO_ROOT/.machine_readable/merge-orchestration/pr-automerge-policy.json" +OUT_DIR="${PWD}/.absence-intake" +ORGS=() +EXECUTE=0 +MAX_NEW="" + +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGS+=("$2"); shift 2 ;; + --policy) POLICY="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + --max-new) MAX_NEW="$2"; shift 2 ;; + --execute) EXECUTE=1; shift ;; + -h|--help) sed -n '2,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +[ ${#ORGS[@]} -eq 0 ] && ORGS=("hyperpolymath" "metadatastician") +command -v gh >/dev/null || { echo "gh is required" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } + +mkdir -p "$OUT_DIR" +PLAN="$OUT_DIR/absence-plan.jsonl" +: > "$PLAN" +log() { printf '%s\n' "$*" >&2; } + +[ -n "$MAX_NEW" ] || MAX_NEW=$(jq -r '.intake_limits.max_new_issues_per_run' "$POLICY") +PER_REPO=$(jq -r '.intake_limits.max_new_issues_per_repo_per_run' "$POLICY") +ROLLUP=$(jq -r '.intake_limits.roll_up_when_class_exceeds' "$POLICY") +MIN_TOPICS=$(jq -r '.absence_rules[] | select(.id == "ABS-002") | .min' "$POLICY") +RUN_DATE=$(date -u +%Y-%m-%d) + +list_repos() { + local org="$1" + gh api "orgs/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) + | {name:.name, description:(.description // ""), topics:(.topics // []), + has_issues:(.has_issues // false)}' 2>/dev/null \ + || gh api "users/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) + | {name:.name, description:(.description // ""), topics:(.topics // []), + has_issues:(.has_issues // false)}' 2>/dev/null +} + +# ─── Enumerate candidates ──────────────────────────────────────────────── +CAND="$OUT_DIR/candidates.jsonl" +: > "$CAND" + +for org in "${ORGS[@]}"; do + list_repos "$org" | jq -c --arg org "$org" '. + {org:$org}' >> "$CAND" +done + +# ABS-001 · empty description · ABS-002 · fewer than MIN_TOPICS topics +jq -c --argjson min "$MIN_TOPICS" ' + select(.has_issues == true) + | if (.description | length) == 0 then {rule:"ABS-001", repo:"\(.org)/\(.name)", detail:"description is empty"} + elif (.topics | length) < $min then {rule:"ABS-002", repo:"\(.org)/\(.name)", + detail:"\(.topics | length) topic(s): \(.topics | join(", "))"} + else empty end +' "$CAND" > "$OUT_DIR/absences.jsonl" + +abs1=$(jq -r 'select(.rule=="ABS-001") | .repo' "$OUT_DIR/absences.jsonl" | wc -l) +abs2=$(jq -r 'select(.rule=="ABS-002") | .repo' "$OUT_DIR/absences.jsonl" | wc -l) +log "candidates: ABS-001 (empty description) ${abs1} · ABS-002 (topics < ${MIN_TOPICS}) ${abs2}" + +# A class over the roll-up threshold becomes ONE listing issue. 43 repositories +# with thin topics is a piece of estate context, not 43 bugs. +if [ "$abs2" -gt "$ROLLUP" ]; then + jq -cn --argjson n "$abs2" --arg min "$MIN_TOPICS" --arg date "$RUN_DATE" \ + --argjson list "$(jq -sc '[.[] | select(.rule == "ABS-002") | .repo]' "$OUT_DIR/absences.jsonl")" \ + '{rule:"ABS-002", repo:"hyperpolymath/standards", kind:"rollup", + detail:(($n|tostring) + " repositories have fewer than " + ($min|tostring) + " topics"), + marker:"", + title_key:"Estate listing:", + title:"Estate listing: \($n) repositories are below the RSR 7-topic minimum", + body:("The RSR standard requires at least " + ($min|tostring) + " topics per repository. " + ($n|tostring) + " active repositories are below it.\n\n" + + "This is one listing rather than " + ($n|tostring) + " issues on purpose: which topics a project should carry is a judgement call, and a hundred parallel judgement calls are not actionable. Set topics per repository at leisure; close this when the count is zero or when the standard changes.\n\n" + + "Repositories:\n\n```\n" + ($list | join("\n")) + "\n```\n\nFiled by `hypatia` ABS-002 on " + $date + ". Deduplicated by this marker: ``")}' \ + >> "$PLAN" + log "ABS-002 rolled up into one listing issue (over the ${ROLLUP} threshold)" +else + jq -c 'select(.rule == "ABS-002")' "$OUT_DIR/absences.jsonl" \ + | while IFS= read -r row; do + repo=$(jq -r '.repo' <<<"$row"); detail=$(jq -r '.detail' <<<"$row") + jq -cn --arg repo "$repo" --arg detail "$detail" --arg date "$RUN_DATE" \ + '{rule:"ABS-002", repo:$repo, kind:"per-repo", detail:$detail, + marker:"", + title_key:"Repository topics are below the RSR minimum of 7", + title:"Repository topics are below the RSR minimum of 7", + body:("This repository has " + $detail + ".\n\n**What would close this:** at least 7 topics covering language, ecosystem, status, licence, domain, maturity and one distinguishing topic.\n\nFiled by `hypatia` ABS-002 on " + $date + ". Deduplicated by this marker: ``")}' \ + >> "$PLAN" + done +fi + +jq -c 'select(.rule == "ABS-001")' "$OUT_DIR/absences.jsonl" \ + | while IFS= read -r row; do + repo=$(jq -r '.repo' <<<"$row") + jq -cn --arg repo "$repo" --arg date "$RUN_DATE" \ + '{rule:"ABS-001", repo:$repo, kind:"per-repo", detail:"description is empty", + marker:"", + title_key:"Repository description is empty", + title:"Repository description is empty", + body:("Repository metadata carries no description.\n\nThis is an absence `hypatia` cannot fix unambiguously: a description has to say what the project *is*, which is a claim only its maintainer can make. Inventing one would be worse than the gap.\n\n**What would close this:** a one-line description in repository settings.\n\nFiled by `hypatia` ABS-001 on " + $date + ". Deduplicated by this marker: ``")}' \ + >> "$PLAN" + done + +plan_count=$(grep -c . "$PLAN" 2>/dev/null || echo 0) + +# ─── Dedup against what is already filed ───────────────────────────────── +: > "$OUT_DIR/to-file.jsonl" +to_file=0 +while IFS= read -r row; do + [ -n "$row" ] || continue + repo=$(jq -r '.repo' <<<"$row"); marker=$(jq -r '.marker' <<<"$row") + title_key=$(jq -r '.title_key // empty' <<<"$row") + # Live issue list, matched locally. The search index lags — an issue filed + # minutes ago can still be missing from it — and a raw HTML-comment marker + # in a query string makes gh stall on the malformed URL. Open issues only: + # a closed issue is a human decision and is never reopened. + existing=$(timeout 60 gh issue list --repo "$repo" --state open --limit 400 \ + --json number,title,body 2>/dev/null \ + | jq -r --arg m "$marker" --arg t "$title_key" ' + [ .[] | select(((.body // "") | contains($m)) + or ($t != "" and (.title | startswith($t)))) ] + | if length > 0 then "\(.[0].number):open" else empty end' 2>/dev/null \ + | head -n1 || true) + + if [ -n "$existing" ]; then + state="${existing##*:}" + jq -c --arg existing "$existing" '. + {already: $existing}' <<<"$row" >> "$OUT_DIR/skipped.jsonl" + log " exists (${state}) ${repo} ${marker}" + else + jq -c --argjson cap "$PER_REPO" '. + {per_repo_cap: $cap}' <<<"$row" >> "$OUT_DIR/to-file.jsonl" + to_file=$((to_file + 1)) + fi +done < "$PLAN" + +log "" +log "─── absence intake ─────────────────────────────────────────────────" +log " planned issues : ${plan_count}" +log " already filed : $(grep -c . "$OUT_DIR/skipped.jsonl" 2>/dev/null || echo 0)" +log " would file : ${to_file} (caps: ${MAX_NEW}/run, ${PER_REPO}/repo/run)" +log " plan : ${PLAN}" + +if [ "$EXECUTE" -ne 1 ]; then + log "" + log " dry run — nothing filed. Pass --execute (with a token that may write" + log " issues on the target repositories) to file them." + exit 2 +fi + +filed=0 +while IFS= read -r row; do + [ -n "$row" ] || continue + [ "$filed" -ge "$MAX_NEW" ] && { log " per-run cap (${MAX_NEW}) reached"; break; } + repo=$(jq -r '.repo' <<<"$row"); title=$(jq -r '.title' <<<"$row"); body=$(jq -r '.body' <<<"$row") + timeout 60 gh issue create --repo "$repo" --title "$title" --body "$body" >/dev/null 2>&1 \ + && { filed=$((filed + 1)); log " filed ${repo}: ${title}"; } \ + || log " could not file ${repo}" +done < "$OUT_DIR/to-file.jsonl" + +log " filed: ${filed}" +exit 0 diff --git a/scripts/sweeps/estate-pin-integrity.sh b/scripts/sweeps/estate-pin-integrity.sh new file mode 100755 index 00000000..de6a2917 --- /dev/null +++ b/scripts/sweeps/estate-pin-integrity.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# estate-pin-integrity.sh — find every denylisted action pin in an estate and, +# with --rewrite, produce the substituted file trees ready for a branch + PR. +# +# WHY THIS EXISTS +# +# github/codeql-action v4.38.1 (1c5b675653bb5c22dbe9b12b556ec555138e09fd) is +# rejected by GitHub at workflow start-up: startup_failure, zero jobs, no logs. +# The estate rolled it back on 2026-09-22. It came back within hours, because +# the rollback was applied as a *relabel*: files across the estate now carry +# +# uses: github/codeql-action/init@1c5b6756... # v4.38.0 +# +# i.e. the poisoned commit wearing a correct-looking comment. A gate that greps +# for the version string sees nothing. A gate that greps for the commit sees +# ~131 files. That is the failure this sweep exists to make impossible. +# +# SAFETY +# +# * Dry run unless --rewrite (which only writes patched trees under --out; +# it never touches a remote). +# * A repair that cannot be proved is never emitted: the substitution must +# leave the file's `uses:` site count unchanged AND remove every denylisted +# token from it. Both are asserted per file before the tree is written. +# * `--via clone` is the authoritative mode (reads the files themselves); +# `--via search` is the fast reconnaissance mode and can lag the index. +# +# EXIT CODES: 0 clean, 1 error, 2 findings present, 3 nothing to do +# +# usage: estate-pin-integrity.sh [--org ORG]... [--policy FILE] [--out DIR] +# [--via search|clone] [--rewrite] +# [--max-repos N] [--sample N] +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +POLICY="$REPO_ROOT/.machine_readable/merge-orchestration/pr-automerge-policy.json" +OUT_DIR="${PWD}/.pin-integrity" +ORGS=() +VIA="search" +REWRITE=0 +MAX_REPOS=0 + +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGS+=("$2"); shift 2 ;; + --policy) POLICY="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + --via) VIA="$2"; shift 2 ;; + --rewrite) REWRITE=1; shift ;; + --max-repos) MAX_REPOS="$2"; shift 2 ;; + -h|--help) sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +[ ${#ORGS[@]} -eq 0 ] && ORGS=("hyperpolymath") +[ -f "$POLICY" ] || { echo "policy not found: $POLICY" >&2; exit 1; } +command -v gh >/dev/null || { echo "gh is required" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } + +mkdir -p "$OUT_DIR" +FINDINGS="$OUT_DIR/pin-findings.jsonl" +PLAN="$OUT_DIR/pin-repair-plan.jsonl" +: > "$FINDINGS" +: > "$PLAN" + +log() { printf '%s\n' "$*" >&2; } + +# `hyperpolymath` is a user account and `metadatastician` is an organisation; +# the two need different endpoints. Trying orgs first keeps the common case to +# one request. +list_repos() { + local org="$1" + gh api "orgs/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | .name' 2>/dev/null \ + || gh api "users/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | .name' 2>/dev/null +} + +# ─── Policy extraction ─────────────────────────────────────────────────── +mapfile -t DENY_SHAS < <(jq -r '.pin_denylist[] | .blocked_shas[]?' "$POLICY") +mapfile -t DENY_VERSIONS < <(jq -r '.pin_denylist[] | .blocked_versions[]?' "$POLICY") + +if [ ${#DENY_SHAS[@]} -eq 0 ] && [ ${#DENY_VERSIONS[@]} -eq 0 ]; then + log "policy carries no blocked pins — nothing to sweep" + exit 3 +fi + +# Every token form a poisoned pin can take: bare SHA, bare version, v-version. +REF_ALTERNATION=$( + { printf '%s\n' "${DENY_SHAS[@]}" + for v in "${DENY_VERSIONS[@]}"; do printf '%s\n' "$v" "v$v"; done + } | sed 's/\./\\./g' | paste -sd'|' - +) + +known_good_sha_for() { + jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_sha' "$POLICY" | head -n1 +} + +known_good_version_for() { + jq -r --arg a "$1" '.pin_denylist[] | select(.action == $a) | .known_good_version' "$POLICY" | head -n1 +} + +# Relabel a pin's inline comment ONLY when the comment leads with a version +# claim. Both estate shapes are covered — `# v4.38.0` and +# `# v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)` — and prose that +# merely mentions a version mid-sentence is never rewritten. +# +# This must stay byte-for-byte identical in spirit to PinIntegrity.relabel/2 in +# lib/rules/pin_integrity.ex. Two readers, one policy, same edit. +relabel_comment() { + local comment="$1" version="$2" + [ -n "$version" ] && [ "$version" != "null" ] || { printf '%s' "$comment"; return; } + local body="${comment#\#}" + local lead="${body%%[![:space:]]*}" + local trimmed="${body#"$lead"}" + if printf '%s' "$trimmed" | grep -qE '^v?[0-9]+(\.[0-9]+)*([[:space:]]|$)'; then + printf '%s%s' "#${lead}" \ + "$(printf '%s' "$trimmed" | sed -E "0,/^v?[0-9]+(\.[0-9]+)*/s//v${version}/")" + else + printf '%s' "$comment" + fi +} + +log "policy $(jq -r '.version' "$POLICY") · blocked SHAs ${#DENY_SHAS[@]} · blocked versions ${#DENY_VERSIONS[@]} · orgs ${ORGS[*]} · via ${VIA}" + +# ─── Phase 1: enumerate candidate files ────────────────────────────────── +HITS="$OUT_DIR/hits.tsv" +: > "$HITS" + +if [ "$VIA" = "search" ]; then + for org in "${ORGS[@]}"; do + for token in "${DENY_SHAS[@]}" "${DENY_VERSIONS[@]}"; do + page=1 + while :; do + n=$(gh api "search/code?q=${token}+org:${org}&per_page=100&page=${page}" \ + --jq '.items[] | "'"${org}"'\t\(.repository.name)\t\(.path)"' 2>/dev/null \ + | tee -a "$HITS" | wc -l) + [ "$n" -lt 100 ] && break + page=$((page + 1)) + [ "$page" -gt 10 ] && break + sleep 2 + done + sleep 2 + done + done + sort -u "$HITS" -o "$HITS" +else + WORK="$OUT_DIR/clones" + mkdir -p "$WORK" + for org in "${ORGS[@]}"; do + mapfile -t reponames < <(list_repos "$org") + [ "$MAX_REPOS" -gt 0 ] && reponames=("${reponames[@]:0:$MAX_REPOS}") + log " cloning ${#reponames[@]} repo(s) in ${org}" + for name in "${reponames[@]}"; do + d="$WORK/$name" + [ -d "$d/.git" ] || git clone --depth 1 --quiet "https://github.com/${org}/${name}.git" "$d" 2>/dev/null || { + log " ! clone failed: ${org}/${name}"; continue; } + while IFS= read -r f; do + printf '%s\t%s\t%s\n' "$org" "$name" "${f#"$d"/}" >> "$HITS" + done < <(grep -rlE "$REF_ALTERNATION" "$d" --include='*.yml' --include='*.yaml' --include='*.lock' --include='*.toml' 2>/dev/null || true) + done + done + sort -u "$HITS" -o "$HITS" +fi + +total_hits=$(wc -l < "$HITS") +log "phase 1: ${total_hits} candidate file(s)" + +if [ "$total_hits" -eq 0 ]; then + echo "clean: no denylisted pin tokens found in ${ORGS[*]}" + exit 0 +fi + +# ─── Phase 2: confirm, classify, and build the repaired tree ───────────── +repairable=0 +flagged=0 + +while IFS=$'\t' read -r org repo path; do + [ -n "${path:-}" ] || continue + + content=$(gh api "repos/${org}/${repo}/contents/${path}?ref=HEAD" --jq '.content' 2>/dev/null | base64 -d 2>/dev/null || true) + if [ -z "$content" ]; then + log " ! cannot read ${org}/${repo}/${path} (binary, missing, or no permission)" + continue + fi + + printf '%s\n' "$content" > "$OUT_DIR/.current" + + sites_before=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.current" || true) + denied_before=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.current" || true) + [ "$denied_before" -eq 0 ] && continue + + # Scope guard: a repair is only meaningful where a pin is *live*. A + # documentation code block quoting the poisoned SHA is a finding, not a + # repair — rewriting prose is how a sweep generates false confidence. + case "$path" in + .github/workflows/*|.github/actions/*) in_scope=1 ;; + */.github/workflows/*|*/.github/actions/*) in_scope=1 ;; + *.lock) in_scope=1 ;; + *) in_scope=0 ;; + esac + + # Rewrite one line at a time so each pin gets the known-good SHA for ITS + # action, and so an action with no known-good pin blocks the whole file. + fixable=1 + actions="" + [ "$in_scope" -eq 0 ] && fixable=0 + : > "$OUT_DIR/.rewritten" + while IFS= read -r line || [ -n "$line" ]; do + if printf '%s\n' "$line" | grep -qE "@(${REF_ALTERNATION})([[:space:]#]|$)"; then + action=$(printf '%s\n' "$line" | sed -E 's/^[[:space:]]*-?[[:space:]]*uses:[[:space:]]*([^@]+)@.*/\1/') + base=$(printf '%s\n' "$action" | awk -F/ '{print $1"/"$2}') + good=$(known_good_sha_for "$base") + actions="${actions}${base} " + if [ -z "$good" ] || [ "$good" = "null" ]; then + fixable=0 + printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" + else + good_version=$(known_good_version_for "$base") + swapped=$(printf '%s\n' "$line" | sed -E "s%@(${REF_ALTERNATION})([[:space:]#]|$)%@${good}\2%g") + head="${swapped%%#*}" + if [ "$head" != "$swapped" ]; then + tail_comment="#${swapped#*#}" + printf '%s%s\n' "$head" "$(relabel_comment "$tail_comment" "$good_version")" >> "$OUT_DIR/.rewritten" + else + printf '%s\n' "$swapped" >> "$OUT_DIR/.rewritten" + fi + fi + else + printf '%s\n' "$line" >> "$OUT_DIR/.rewritten" + fi + done < "$OUT_DIR/.current" + + sites_after=$(grep -cE '^[[:space:]]*-?[[:space:]]*uses:' "$OUT_DIR/.rewritten" || true) + denied_after=$(grep -cE "@(${REF_ALTERNATION})([[:space:]#]|$)" "$OUT_DIR/.rewritten" || true) + + status="flag" + reason="owner_review" + if [ "$fixable" -eq 1 ] && [ "$sites_after" -eq "$sites_before" ] && [ "$denied_after" -eq 0 ] \ + && ! cmp -s "$OUT_DIR/.current" "$OUT_DIR/.rewritten"; then + status="repairable" + reason="substitution" + elif [ "$in_scope" -eq 0 ]; then + reason="out_of_scope_file" + elif [ "$fixable" -eq 0 ]; then + reason="no_known_good_pin_for_action" + elif [ "$sites_after" -ne "$sites_before" ]; then + reason="repair_would_change_pin_site_count" + fi + + jq -cn \ + --arg repo "${org}/${repo}" --arg path "$path" --arg status "$status" \ + --arg reason "$reason" --arg actions "$(printf '%s' "$actions" | xargs || true)" \ + --argjson denied "$denied_before" --argjson sites "$sites_before" \ + '{repo:$repo, path:$path, rule:"PI001", status:$status, repair:$reason, + actions:($actions|split(" ")|map(select(.!=""))), + denylisted_pin_sites:$denied, uses_sites:$sites}' >> "$FINDINGS" + + if [ "$status" = "repairable" ]; then + repairable=$((repairable + 1)) + if [ "$REWRITE" -eq 1 ]; then + mkdir -p "$OUT_DIR/patched/${repo}/$(dirname "$path")" + cp "$OUT_DIR/.rewritten" "$OUT_DIR/patched/${repo}/${path}" + fi + else + flagged=$((flagged + 1)) + log " flagged ${org}/${repo}/${path}: ${reason}" + fi +done < "$HITS" + +# One repair PR per repository. +if [ "$repairable" -gt 0 ]; then + jq -src ' + map(select(.status == "repairable")) + | group_by(.repo) + | .[] + | {repo: .[0].repo, + files: length, + paths: [.[].path], + action: "pin_rollback_denylisted", + method: "squash", + safety: "arm_auto", + route: "Patch-Bridge", + branch: "hypatia/pin-integrity-known-good", + rule: "PI001"} + ' "$FINDINGS" > "$PLAN" +fi + +log "" +log "─── pin-integrity sweep ────────────────────────────────────────────" +log " candidate files : ${total_hits}" +log " mechanically repairable : ${repairable}" +log " needs owner review : ${flagged}" +log " repos with a repair ready : $(grep -c . "$PLAN" 2>/dev/null || echo 0)" +log " findings : ${FINDINGS}" +log " repair plan : ${PLAN}" +[ "$REWRITE" -eq 1 ] && log " repaired trees : ${OUT_DIR}/patched" +log " NOTE: this sweep never writes to a remote. Branch + PR creation is" +log " the token-bearing actuator's job — docs/operations/estate-automerge.adoc" + +exit 2 diff --git a/scripts/sweeps/estate-pr-automerge.sh b/scripts/sweeps/estate-pr-automerge.sh new file mode 100755 index 00000000..f1619dab --- /dev/null +++ b/scripts/sweeps/estate-pr-automerge.sh @@ -0,0 +1,561 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# estate-pr-automerge.sh — decide, for every open PR in an estate, whether it +# is an unambiguous bump/chore/pin that may simply be merged and its branch +# deleted; whether it carries a poisoned pin that must be excised first; or +# whether it is one of the cases that only *looks* routine. +# +# WHY THIS IS NOT A TITLE MATCHER +# +# On 2026-09-26 the estate had 34 open PRs, every one of them a +# `chore(deps): bump …` from dependabot. Every one of them looked safe. +# +# * 24 of them re-introduce github/codeql-action v4.38.1 — the commit +# GitHub rejects at workflow start-up. Merging any of them breaks CodeQL +# and the Hypatia scan on that repository. This is not hypothetical: +# nexia-list#107 and dictask#65 both merged one, and both repositories +# are red on main today. +# * at least two of them hide a *major* bump inside a "grouped" chore +# (`actions/checkout` v4.1.7 → v7.0.1, labelled `# v4` on both sides). +# +# A title matcher merges all 34. This classifier merges 9 and explains the +# other 25. That difference is the whole point of the exercise. +# +# WHAT MAKES A PR ELIGIBLE (all must hold) +# +# 1. authored by a dependency bot +# 2. every changed line is either a `uses:` pin line, a lockfile line, or +# in a non-code metadata path +# 3. no denylisted token appears in an added line +# 4. the pin delta is *resolvable* and is not a major-version increase — +# resolved from the action's own tag list, not from the inline comment, +# because the comments in this estate demonstrably lie +# +# Anything failing (4) is not rejected as unsafe; it is reported as +# unverifiable. "Cannot prove safe" and "proved unsafe" are different verdicts +# and the manifest keeps them different. +# +# EXIT CODES: 0 nothing to decide, 1 error, 2 decisions written +# +# usage: estate-pr-automerge.sh [--org ORG]... [--policy FILE] [--out DIR] +# [--max-repos N] [--cache DIR] [--execute] +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +POLICY="$REPO_ROOT/.machine_readable/merge-orchestration/pr-automerge-policy.json" +DECISIONS_IN="" +OUT_DIR="${PWD}/.pr-automerge" +CACHE_DIR="${PWD}/.pr-automerge/cache" +ORGS=() +MAX_REPOS=0 +EXECUTE=0 + +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGS+=("$2"); shift 2 ;; + --policy) POLICY="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + --cache) CACHE_DIR="$2"; shift 2 ;; + --max-repos) MAX_REPOS="$2"; shift 2 ;; + --execute) EXECUTE=1; shift ;; + --from) DECISIONS_IN="$2"; shift 2 ;; + -h|--help) sed -n '2,45p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +[ ${#ORGS[@]} -eq 0 ] && ORGS=("hyperpolymath") +[ -f "$POLICY" ] || { echo "policy not found: $POLICY" >&2; exit 1; } +command -v gh >/dev/null || { echo "gh is required" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } + +mkdir -p "$OUT_DIR" "$CACHE_DIR" +DECISIONS="$OUT_DIR/pr-decisions.jsonl" +: > "$DECISIONS" + +log() { printf '%s\n' "$*" >&2; } + +# `hyperpolymath` is a user account and `metadatastician` is an organisation; +# the two need different endpoints. Trying orgs first keeps the common case to +# one request. +# Emits `namearchived`. Archived repositories are INCLUDED: a dependabot +# PR left open on an archived repo is unmergeable by construction and is one of +# the "chores that are obviously going to be merged some day" that never will. +# Knowing about it is the difference between a backlog and an explanation. +list_repos() { + local org="$1" + gh api "orgs/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.fork == false and .disabled == false) | "\(.name)\t\(.archived)"' 2>/dev/null \ + || gh api "users/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.fork == false and .disabled == false) | "\(.name)\t\(.archived)"' 2>/dev/null +} + +mapfile -t DENY_SHAS < <(jq -r '.pin_denylist[] | .blocked_shas[]?' "$POLICY") +mapfile -t DENY_VERS < <(jq -r '.pin_denylist[] | .blocked_versions[]?' "$POLICY") +mapfile -t DEP_BOTS < <(jq -r '.dependency_bots[]' "$POLICY") +mapfile -t LOCKFILES < <(jq -r '.lockfile_names[]' "$POLICY") + +REF_ALT=$( + { printf '%s\n' "${DENY_SHAS[@]}" + for v in "${DENY_VERS[@]}"; do printf '%s\n' "$v" "v$v"; done + } | sed 's/\./\\./g' | paste -sd'|' - +) + +is_dep_bot() { + local a="$1" + for b in "${DEP_BOTS[@]}"; do [ "$a" = "$b" ] && return 0; done + return 1 +} + +is_lockfile() { + local base + base="$(basename "$1")" + for l in "${LOCKFILES[@]}"; do [ "$base" = "$l" ] && return 0; done + return 1 +} + +is_metadata_path() { + case "$1" in + *.md|*.adoc|*.rst|*.txt|CODEOWNERS|.gitattributes|.editorconfig) return 0 ;; + docs/*|doc/*|.github/ISSUE_TEMPLATE/*) return 0 ;; + *) return 1 ;; + esac +} + +# Licence/SPDX files are metadata *paths* but never a metadata *change*: +# the decision contract routes them to a null authority (owner review). +# Auto-merging a licence edit would let the robot make a claim about what +# the project is. +is_licence_path() { + local base + base="$(basename "$1")" + case "$base" in + *LICENSE*|*LICENCE*|*COPYING*|*NOTICE*|*SPDX*) return 0 ;; + esac + case "$1" in + LICENSES/*) return 0 ;; + esac + return 1 +} + +# ─── SHA → version, from the action's own tag list ─────────────────────── +# The estate's inline comments were wrong often enough (a `# v4` label on the +# v7.0.1 commit) that they cannot be the source of truth. This resolves +# against the upstream tags and caches per action. +resolve_version() { + local action="$1" sha="$2" + local cache="$CACHE_DIR/$(printf '%s' "$action" | tr '/' '_').json" + if [ ! -s "$cache" ]; then + gh api "repos/${action}/tags?per_page=100" \ + --jq '[.[] | {tag: .name, sha: .commit.sha}]' > "$cache" 2>/dev/null || echo '[]' > "$cache" + fi + jq -r --arg sha "$sha" ' + [.[] | select(.sha == $sha) | .tag] | first // empty + ' "$cache" 2>/dev/null | sed 's/^v//' +} + +major_of() { printf '%s' "${1:-}" | sed -E 's/^v?([0-9]+).*/\1/' ; } + +# ─── Enumerate open PRs ────────────────────────────────────────────────── +PRS="$OUT_DIR/open-prs.jsonl" +: > "$PRS" + +for org in "${ORGS[@]}"; do + mapfile -t reponames < <(list_repos "$org") + [ "$MAX_REPOS" -gt 0 ] && reponames=("${reponames[@]:0:$MAX_REPOS}") + log "scanning ${#reponames[@]} repo(s) in ${org} for open PRs" + + for row in "${reponames[@]}"; do + name="${row%%$'\t'*}" + archived="${row##*$'\t'}" + # NB: `gh api --jq` does not accept --arg; the org is injected with a + # real jq stage instead. (Silently swallowing this is how a sweep reports + # "0 open PRs" against an estate with 35 of them.) + gh api "repos/${org}/${name}/pulls?state=open&per_page=100" 2>/dev/null \ + | jq -c --arg org "$org" --argjson arch "${archived:-false}" '.[] | {org:$org, repo:.base.repo.name, number:.number, title:.title, + repo_archived:$arch, + author:.user.login, draft:.draft, mergeable:.mergeable, mergeable_state:.mergeable_state, + created:.created_at, head_sha:.head.sha, head_ref:.head.ref, base_ref:.base.ref, + body:(.body // ""), changed_files:.changed_files, labels:[.labels[].name]}' 2>/dev/null >> "$PRS" || true + done +done + +total=$(wc -l < "$PRS") +log "found ${total} open PR(s)" +open_repos=$(gh api "search/issues?q=org:${ORGS[0]}+is:pr+is:open&per_page=1" --jq '.total_count' 2>/dev/null || echo "?") +log " (GitHub search index reports ${open_repos} open PR(s) for ${ORGS[0]}; a large gap means the enumeration is broken)" +if [ "$total" -eq 0 ]; then + echo "nothing to decide" + exit 0 +fi + +# ─── Classify ──────────────────────────────────────────────────────────── +auto=0; excise=0; close_poison=0; flagged=0 + +while IFS= read -r pr; do + org=$(jq -r '.org' <<<"$pr") + repo=$(jq -r '.repo' <<<"$pr") + num=$(jq -r '.number' <<<"$pr") + title=$(jq -r '.title' <<<"$pr") + author=$(jq -r '.author' <<<"$pr") + draft=$(jq -r '.draft' <<<"$pr") + mstate=$(jq -r '.mergeable_state' <<<"$pr") + head_sha=$(jq -r '.head_sha' <<<"$pr") + created=$(jq -r '.created' <<<"$pr") + base_ref=$(jq -r '.base_ref' <<<"$pr") + repo_archived=$(jq -r '.repo_archived // false' <<<"$pr") + body=$(jq -r '.body // ""' <<<"$pr") + + # Dependabot states its own version delta in the body: + # "Updates `haskell-actions/setup` from 2.12.0 to 2.12.1" + # That claim is used as a SECOND source, never as the only one. When the diff + # and the claim disagree the PR is flagged, not merged. + # Grouped PRs write: Updates `owner/action` from A to B + # Single-dep PRs write: Bumps [name](https://…) from A to B. + # Both are captured; neither is trusted alone. A body can be edited by + # anyone who can edit the PR, so it is corroboration, not authority. + body_deltas=$(grep -oE '(Updates|Bumps) (\[)?`?[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*`?(\])?(\([^)]*\))? from [0-9][^ ]* to [0-9][^ ]*' <<<"$body" \ + | sed -E 's/^(Updates|Bumps) (\[)?`?([A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)*)`?(\])?(\([^)]*\))? from ([^ ]+) to (.*)$/\3|\7|\8/; s/\.+$//' \ + | jq -Rsc 'split("\n") | map(select(. != "")) | map(split("|") | {action: .[0], from: .[1], to: .[2]})' 2>/dev/null || echo '[]') + + body_major=0 + while IFS= read -r c; do + [ -n "$c" ] || continue + cf=$(jq -r '.from' <<<"$c"); ct=$(jq -r '.to' <<<"$c") + [ "$(major_of "$cf")" != "$(major_of "$ct")" ] && body_major=1 + done < <(jq -c '.[]' <<<"$body_deltas" 2>/dev/null || true) + + now_epoch=$(date -u +%s) + created_epoch=$(date -u -d "$created" +%s 2>/dev/null || echo "$now_epoch") + age_days=$(( (now_epoch - created_epoch) / 86400 )) + + full="$org/$repo" + patches=$(gh api "repos/${full}/pulls/${num}/files?per_page=100" \ + --jq '.[] | {filename:.filename, patch:(.patch // "")}' 2>/dev/null || echo '') + + # file-level analysis + files_total=$(jq -c . <<<"$patches" | wc -l) + poison_hits=0; nonpin_change=0; lock_only=1; meta_only=1; licence_touch=0; pin_files=0 + declare -a deltas=() + + while IFS= read -r fobj; do + [ -n "$fobj" ] || continue + fn=$(jq -r '.filename' <<<"$fobj") + fp=$(jq -r '.patch' <<<"$fobj") + + is_lockfile "$fn" || lock_only=0 + is_metadata_path "$fn" || meta_only=0 + is_licence_path "$fn" && licence_touch=1 + + # Every +/- content line, ignoring hunk headers and file markers. + content_lines=$(grep -E '^[-+]' <<<"$fp" | grep -vE '^(\+\+\+|---)' || true) + [ -z "$content_lines" ] && content_lines="" + + added=$(grep -E '^\+' <<<"$content_lines" | grep -vE '^\+\+\+' || true) + removed=$(grep -E '^-' <<<"$content_lines" | grep -vE '^---' || true) + + if [ -n "$added" ] && grep -qE "@(${REF_ALT})([[:space:]#]|$)" <<<"$added"; then + poison_hits=$((poison_hits + 1)) + fi + + # Is every changed line a `uses:` line? + if grep -qvE '^\s*-?\s*uses:' <<<"$content_lines" 2>/dev/null; then + : + fi + only_uses=1 + while IFS= read -r cl; do + [ -n "$cl" ] || continue + printf '%s' "$cl" | grep -qE '^[-+]\s*-?\s*uses:' || only_uses=0 + done <<<"$content_lines" + [ "$only_uses" -eq 1 ] && [ -n "$content_lines" ] && pin_files=$((pin_files + 1)) + [ "$only_uses" -eq 0 ] && nonpin_change=$((nonpin_change + 1)) + + # pin deltas: pair removed and added `uses:` lines per action + while IFS= read -r rline; do + [ -n "$rline" ] || continue + raction=$(sed -E 's/^[-+]\s*-?\s*uses:\s*([^@]+)@.*/\1/' <<<"$rline") + rref=$(sed -E 's/^[-+]\s*-?\s*uses:\s*[^@]+@([^[:space:]#]+).*/\1/' <<<"$rline") + base="${raction%%/*}/$(printf '%s' "$raction" | cut -d/ -f2)" + aline=$(grep -E "^\+.*uses:\s*${raction}@" <<<"$added" | head -n1 || true) + [ -n "$aline" ] || continue + aref=$(sed -E 's/^\+.*uses:\s*[^@]+@([^[:space:]#]+).*/\1/' <<<"$aline") + [ "$rref" = "$aref" ] && continue + oldv=$(resolve_version "$base" "$rref") + newv=$(resolve_version "$base" "$aref") + deltas+=("${base}|${oldv:-?}|${newv:-?}") + done <<<"$removed" + done <<<"$(jq -c . <<<"$patches")" + + delta_json=$(printf '%s\n' "${deltas[@]:-}" | jq -Rsc 'split("\n") | map(select(. != "")) | map(split("|") | {action: .[0], from: .[1], to: .[2]})') + + # Version delta. Three sources, in decreasing authority: + # 1. the diff — which actions changed, and to which refs (ground truth) + # 2. upstream tags — what version a ref is (resolved via the action's tags) + # 3. the PR body — dependabot's own claim, used only to fill a gap + # A gap in (2) filled by (3) is recorded with source `body-claim`, so a + # reviewer can see exactly which claim gated the merge. Where (2) and (3) + # disagree the PR is flagged: a disagreement means one of them is wrong and + # the estate has already been burned once by exactly that. + major_delta=0; unresolved=0; conflict=0 + resolved_deltas=() + + while IFS= read -r d; do + [ -n "$d" ] || continue + daction=$(jq -r '.action' <<<"$d") + from=$(jq -r '.from' <<<"$d"); to=$(jq -r '.to' <<<"$d") + source="tags" + + shorter=$(printf '%s' "$daction" | awk -F/ '{print $NF}') + bclaim=$(jq -c --arg a "$daction" --arg b "$shorter" '.[] | select(.action == $a or .action == $b)' \ + <<<"$body_deltas" 2>/dev/null | head -n1 || true) + + if [ -n "$bclaim" ]; then + bfrom=$(jq -r '.from' <<<"$bclaim"); bto=$(jq -r '.to' <<<"$bclaim") + if [ "$from" != "?" ] && [ "$to" != "?" ]; then + { [ "$from" != "$bfrom" ] || [ "$to" != "$bto" ]; } && conflict=1 + else + [ "$from" = "?" ] && from="$bfrom" + [ "$to" = "?" ] && to="$bto" + source="body-claim" + fi + fi + + if [ "$from" = "?" ] || [ "$to" = "?" ]; then + unresolved=$((unresolved + 1)) + resolved_deltas+=("${daction}|?|?|unresolved") + continue + fi + + [ "$(major_of "$from")" != "$(major_of "$to")" ] && major_delta=1 + resolved_deltas+=("${daction}|${from}|${to}|${source}") + done < <(jq -c '.[]' <<<"$delta_json" 2>/dev/null || true) + + # A lockfile-only PR has no `uses:` lines to attach a delta to, so the claim + # list itself is the delta list. Without a claim there is nothing to check, + # and "nothing to check" is not a licence to merge. + if [ "$lock_only" -eq 1 ]; then + [ "$body_major" -eq 1 ] && major_delta=1 + if [ "$(jq 'length' <<<"$body_deltas" 2>/dev/null || echo 0)" -eq 0 ]; then + unresolved=1 + else + while IFS= read -r c; do + [ -n "$c" ] || continue + resolved_deltas+=("$(jq -r '.action' <<<"$c")|$(jq -r '.from' <<<"$c")|$(jq -r '.to' <<<"$c")|body-claim") + done < <(jq -c '.[]' <<<"$body_deltas" 2>/dev/null || true) + fi + fi + + delta_json=$(printf '%s\n' "${resolved_deltas[@]:-}" \ + | jq -Rsc 'split("\n") | map(select(. != "")) | map(split("|") | {action: .[0], from: .[1], to: .[2], source: .[3]})') + + # ── Verdict ──────────────────────────────────────────────────────────── + change_class="chore"; change_level="object"; safety="flag"; method="squash" + pool="P3"; route="rhodibot"; disposition="flag"; blocked_by="" + + if [ "$repo_archived" = "true" ]; then + disposition="close_archived_repo" + blocked_by="repository_is_archived_cannot_merge" + safety="flag"; route="rhodibot"; pool="P3" + elif ! is_dep_bot "$author"; then + blocked_by="not_a_dependency_bot" + elif [ "$draft" = "true" ]; then + blocked_by="draft" + elif [ "$mstate" = "dirty" ] || [ "$mstate" = "CONFLICTING" ]; then + blocked_by="merge_conflict" + elif [ "$poison_hits" -gt 0 ] && [ "$files_total" -gt 0 ]; then + change_class="bump" + if [ "$major_delta" -eq 1 ]; then + # Nothing in this PR is mergeable as-is: it is poisoned AND it carries + # major bumps. Closing is the only correct disposition, and the reason + # must say so rather than blaming the pin alone. + disposition="close_poison_and_majors" + blocked_by="introduces_denylisted_pin_and_major_bumps" + elif [ "$nonpin_change" -eq 0 ] && [ "$lock_only" -eq 0 ]; then + disposition="close_poison_only" + blocked_by="introduces_denylisted_pin" + else + disposition="excise_poison_then_merge" + blocked_by="introduces_denylisted_pin_alongside_wanted_updates" + fi + safety="flag"; route="Patch-Bridge"; pool="P1" + elif [ "$major_delta" -eq 1 ]; then + change_class="bump"; disposition="flag"; safety="flag" + blocked_by="major_version_delta" + route="rhodibot"; pool="P2" + elif [ "$conflict" -eq 1 ]; then + change_class="bump"; disposition="flag"; safety="flag" + blocked_by="version_claims_conflict" + route="rhodibot"; pool="P2" + elif [ "$unresolved" -gt 0 ]; then + change_class="bump"; disposition="flag"; safety="flag" + blocked_by="pin_delta_unresolvable" + route="rhodibot"; pool="P2" + elif [ "$pin_files" -eq 0 ] && [ "$lock_only" -eq 0 ] && + jq -r '.filename' <<<"$patches" 2>/dev/null \ + | grep -qE '(^|/)(package\.json|Cargo\.toml|Project\.toml|mix\.exs|go\.mod|pyproject\.toml|composer\.json|Gemfile)$'; then + change_class="bump"; disposition="flag"; safety="flag" + blocked_by="dependency_manifest_not_lockfile" + route="rhodibot"; pool="P2" + elif [ "$lock_only" -eq 1 ]; then + change_class="bump"; disposition="auto_merge"; safety="arm_auto" + route="Patch-Bridge"; pool="P2" + blocked_by="awaiting_required_checks" + elif [ "$pin_files" -gt 0 ] && [ "$nonpin_change" -eq 0 ]; then + change_class="bump"; disposition="auto_merge"; safety="arm_auto" + route="Patch-Bridge"; pool="P2" + blocked_by="awaiting_required_checks" + elif [ "$licence_touch" -eq 1 ]; then + change_class="chore"; disposition="flag"; safety="flag" + route="rhodibot"; pool="P2" + blocked_by="touches_licence_requires_owner_review" + elif [ "$meta_only" -eq 1 ]; then + change_class="chore"; disposition="auto_merge"; safety="arm_auto" + route="rhodibot"; pool="P3" + blocked_by="awaiting_required_checks" + else + blocked_by="not_unambiguously_classifiable" + fi + + # Workflow edits are `meta` by the decision contract unless the change is a + # pure pin substitution. The actuator re-proves this; the brain states it. + if [ "$pin_files" -gt 0 ] && [ "$nonpin_change" -eq 0 ]; then + meta_exemption="MGX-001" + else + meta_exemption="" + if grep -q '\.github/workflows/' <<<"$(jq -r '.[].filename' <<<"$patches" 2>/dev/null)"; then + change_level="meta" + [ "$safety" = "arm_auto" ] && { safety="flag"; disposition="flag"; blocked_by="meta_no_pin_exemption"; } + fi + fi + + case "$disposition" in + auto_merge) auto=$((auto + 1)) ;; + excise_poison_then_merge) excise=$((excise + 1)) ;; + close_poison_only|close_poison_and_majors) close_poison=$((close_poison + 1)) ;; + *) flagged=$((flagged + 1)) ;; + esac + + vetoes='[]' + if [ "$poison_hits" -gt 0 ]; then + vetoes=$(jq -cn --arg r "$blocked_by" '[{bot:"Patch-Bridge", reason:$r}]') + fi + if [ "$change_level" = "meta" ]; then + vetoes=$(jq -cn --argjson v "$vetoes" '$v + [{bot:"hypatia", reason:"change_level=meta"}]') + fi + + jq -cn \ + --arg repo "$full" --argjson n "$num" --arg head "$head_sha" --arg base "$base_ref" \ + --arg author "$author" --arg author_kind "dependabot" \ + --arg cc "$change_class" --arg cl "$change_level" --arg route "$route" \ + --arg method "$method" --arg safety "$safety" --arg pool "$pool" \ + --arg disposition "$disposition" --arg blocked "$blocked_by" \ + --argjson age "$age_days" --argjson delta "$delta_json" \ + --argjson vetoes "$vetoes" --argjson poison "$poison_hits" \ + --argjson files "$files_total" --arg mex "$meta_exemption" \ + --arg version "$(jq -r '.version' "$POLICY")" \ + '{pr:{repo:$repo, number:$n, head_sha:$head, base:$base, author:$author, author_kind:$author_kind}, + change_class:$cc, change_level:$cl, + route:{authority_bot:$route, contributing_bots:["hypatia"]}, + method:$method, method_basis:"repo-default", safety:$safety, pool:$pool, + confidence:null, + attestations:[{bot:"hypatia", verdict:"approve", confidence:0.9, + rationale:"classified from the diff, not the title"}], + vetoes:$vetoes, clamped_by:(if $vetoes == [] then null else "veto" end), + rationale:("policy " + $version + " · " + $disposition + " · blocked_by=" + $blocked), + disposition:$disposition, blocked_by:$blocked, age_days:$age, + pin_delta:$delta, denylisted_hits:$poison, files_changed:$files, + meta_guard_exemption:$mex, + timestamp:(now | todate)}' >> "$DECISIONS" + +done < "$PRS" + +log "" +log "─── PR automerge decisions ─────────────────────────────────────────" +log " open PRs examined : ${total}" +log " auto_merge : ${auto}" +log " excise_then_merge : ${excise}" +log " close_and_delete : ${close_poison}" +log " flagged for review : ${flagged}" +log " decisions : ${DECISIONS}" + +if [ "$EXECUTE" -ne 1 ]; then + exit 2 +fi + +# ─── Actuator ──────────────────────────────────────────────────────────── +# Two rules, neither negotiable: +# +# 1. Re-verify, never trust. Every decision is re-derived from the PR as it +# is NOW: the head commit must be the one the decision was made about, +# and the denylist check is re-run against the live diff. A manifest is +# evidence, not authority — the same reasoning as `--auto` merges in +# estate-rescan.yml. +# 2. Delete the branch. A merged or closed chore whose branch lingers is +# the same backlog one layer down. +log "" +log "─── actuating (re-verifying every decision against the live PR) ────" + +MANIFEST="${DECISIONS_IN:-$DECISIONS}" +applied=0; skipped=0 + +while IFS= read -r d; do + repo=$(jq -r '.pr.repo' <<<"$d"); num=$(jq -r '.pr.number' <<<"$d") + want_sha=$(jq -r '.pr.head_sha' <<<"$d"); disp=$(jq -r '.disposition' <<<"$d") + blocked=$(jq -r '.blocked_by' <<<"$d") + + live_sha=$(gh api "repos/${repo}/pulls/${num}" --jq '.head.sha' 2>/dev/null || echo "") + if [ "$live_sha" != "$want_sha" ]; then + log " skip ${repo}#${num}: head moved (${want_sha:0:8} → ${live_sha:0:8}) — decision is stale" + skipped=$((skipped + 1)); continue + fi + + # Independent re-proof of the pin verdict, from the live diff. + live_added=$(gh api "repos/${repo}/pulls/${num}/files?per_page=100" --jq '.[].patch // ""' 2>/dev/null | grep -E '^\+' | grep -vE '^\+\+\+' || true) + live_poison=0 + grep -qE "@(${REF_ALT})([[:space:]#]|$)" <<<"$live_added" && live_poison=1 + + case "$disp" in + auto_merge) + if [ "$live_poison" -eq 1 ]; then + log " REFUSE ${repo}#${num}: live diff carries a denylisted pin — the manifest is wrong" + skipped=$((skipped + 1)); continue + fi + gh pr merge "$num" --repo "$repo" --squash --auto --delete-branch >/dev/null 2>&1 \ + && { log " merged ${repo}#${num} (auto, branch deleted)"; applied=$((applied + 1)); } \ + || { log " could not queue ${repo}#${num}"; skipped=$((skipped + 1)); } + ;; + close_poison_only|close_poison_and_majors|close_archived_repo) + if [ "$disp" = "close_poison_only" ] && [ "$live_poison" -eq 0 ]; then + log " REFUSE ${repo}#${num}: manifest says poisoned, live diff disagrees" + skipped=$((skipped + 1)); continue + fi + body="Closed by hypatia's estate sweep: ${blocked}. + +This pull request cannot be merged as it stands. It would put a pin on the +estate denylist (\`pin_denylist\` in \`pr-automerge-policy.json\`) back into the +tree, so merging it would break the workflows it touches rather than update +them. Nothing here is lost: dependabot re-raises the wanted bumps on its next +run, and \`exclude-patterns\` in \`.github/dependabot.yml\` stops the poisoned +one being raised again. + +Reopen freely if this disposition is wrong — the sweep is a robot, the +repository is yours." + gh pr comment "$num" --repo "$repo" --body "$body" >/dev/null 2>&1 || true + gh pr close "$num" --repo "$repo" --delete-branch >/dev/null 2>&1 \ + && { log " closed ${repo}#${num} (branch deleted)"; applied=$((applied + 1)); } \ + || { log " could not close ${repo}#${num}"; skipped=$((skipped + 1)); } + ;; + *) + skipped=$((skipped + 1)) + ;; + esac +done < "$MANIFEST" + +log "" +log " applied : ${applied}" +log " skipped : ${skipped} (flags, stale heads, and refused re-verifications)" +exit 0 diff --git a/scripts/sweeps/estate-stats.sh b/scripts/sweeps/estate-stats.sh new file mode 100755 index 00000000..2594bc08 --- /dev/null +++ b/scripts/sweeps/estate-stats.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# estate-stats.sh — build the single estate-wide statistics artifact that the +# private farm dashboard reads. +# +# WHY +# +# The complaint was specific: after the issue count ran away, the one thing +# that became impossible to see was the *state of the estate* — how many PRs +# are open and what class they are, which repositories have no tests or no +# benches at all, which suites are failing, which benches are over their +# limit. Those numbers existed in five places and were displayed in none. +# +# This emits ONE file, `estate-stats.json`, with a frozen schema version, the +# run time, and — critically — an explicit provenance per section saying where +# each number came from and whether it was actually available. A metric that +# could not be measured is reported as unavailable, never as 0. (An earlier +# generation of this estate's dashboards showed zeros for data that had simply +# never been fetched, which is how "green" became meaningless.) +# +# SOURCES +# +# repositories, pull requests, issues GitHub API + a decisions manifest +# security weak points --scans (panic-attack scan store) +# test surface, failing suites --checks (GitHub Actions API) +# bench coverage / over-limit ratio no producer in the estate yet — +# reported unavailable with the +# contract it must satisfy +# +# The `paging` block at the top level evaluates every threshold in +# policy.stats_thresholds against the measured value, so the dashboard does +# not have to know which number is "serious". A metric with no source is +# `unavailable` there too — a threshold cannot be breached by data that was +# never collected. +# +# Consumer contract (see pr-automerge-policy.json → stats_output): +# producer: hypatia docs/status/estate-stats.json +# consumer: hyperpolymath/.git-private-farm +# metadatastician/berrywiki/data/estate-stats.json +# +# EXIT CODES: 0 ok, 1 error +# +# usage: estate-stats.sh [--org ORG]... [--out FILE] [--decisions FILE] +# [--scans DIR] [--checks] [--max-repos N] +# [--max-issues N] +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +POLICY="$REPO_ROOT/.machine_readable/merge-orchestration/pr-automerge-policy.json" +OUT="${PWD}/estate-stats.json" +DECISIONS="" +SCANS_DIR="" +ORGS=() +MAX_ISSUES=1000 +CHECKS=0 +MAX_REPOS=0 + +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGS+=("$2"); shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --decisions) DECISIONS="$2"; shift 2 ;; + --scans) SCANS_DIR="$2"; shift 2 ;; + --checks) CHECKS=1; shift ;; + --max-repos) MAX_REPOS="$2"; shift 2 ;; + --max-issues) MAX_ISSUES="$2"; shift 2 ;; + -h|--help) sed -n '2,54p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +[ ${#ORGS[@]} -eq 0 ] && ORGS=("hyperpolymath" "metadatastician") +command -v gh >/dev/null || { echo "gh is required" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } +log() { printf '%s\n' "$*" >&2; } + +RUN_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) +policy_version=$(jq -r '.version' "$POLICY") +TMP=$(mktemp -d) +trap 'rm -rf "$TMP"' EXIT + +list_repos() { + local org="$1" + gh api "orgs/${org}/repos?per_page=100" --paginate \ + --jq '.[] | {name:.name, archived:.archived, fork:.fork, disabled:.disabled, + description:(.description // ""), topics:(.topics // []), + default_branch:(.default_branch // "main"), + pushed_at:(.pushed_at // ""), has_issues:(.has_issues // false)}' 2>/dev/null \ + || gh api "users/${org}/repos?per_page=100" --paginate \ + --jq '.[] | {name:.name, archived:.archived, fork:.fork, disabled:.disabled, + description:(.description // ""), topics:(.topics // []), + default_branch:(.default_branch // "main"), + pushed_at:(.pushed_at // ""), has_issues:(.has_issues // false)}' 2>/dev/null +} + +# ─── Repositories + absences ───────────────────────────────────────────── +: > "$TMP/repos.jsonl" +for org in "${ORGS[@]}"; do + list_repos "$org" | jq -c --arg org "$org" '. + {org:$org}' >> "$TMP/repos.jsonl" +done +repo_count=$(jq -s 'length' "$TMP/repos.jsonl") +log "repos: ${repo_count}" + +MIN_TOPICS=$(jq -r '.absence_rules[] | select(.id == "ABS-002") | .min' "$POLICY") + +jq -s --argjson min "$MIN_TOPICS" ' + { + total: length, + active: (map(select(.archived == false)) | length), + archived: (map(select(.archived == true)) | length), + empty_description: { + count: (map(select(.archived == false and (.description | length) == 0)) | length), + repos: [.[] | select(.archived == false and (.description | length) == 0) | "\(.org)/\(.name)"] + }, + topics_below_min: { + min: $min, + count: (map(select(.archived == false and (.topics | length) < $min)) | length), + repos: [map(select(.archived == false and (.topics | length) < $min)) + | sort_by(.topics | length) | .[] | {repo: "\(.org)/\(.name)", topics: (.topics | length)}] + } + } +' "$TMP/repos.jsonl" > "$TMP/absences.json" + +# ─── Open PRs by class, from the decisions manifest ────────────────────── +if [ -z "$DECISIONS" ]; then + for cand in "${PWD}/.pr-automerge/pr-decisions.jsonl" "$REPO_ROOT/.pr-automerge/pr-decisions.jsonl"; do + [ -s "$cand" ] && DECISIONS="$cand" && break + done +fi + +if [ -n "$DECISIONS" ] && [ -s "$DECISIONS" ]; then + jq -s ' + { + available: true, + source: "pr-decisions.jsonl", + total: length, + by_disposition: (group_by(.disposition) | map({key: .[0].disposition, value: length}) | from_entries), + by_blocked_by: (group_by(.blocked_by) | map({key: .[0].blocked_by, value: length}) | from_entries), + by_repo_archived: (map(select(.pr.repo_archived == true)) | length), + oldest_age_days: (map(.age_days) | max // 0), + median_age_days: (map(.age_days) | sort | (if length == 0 then 0 else .[length / 2 | floor] end)), + poison_carrying: (map(select(.denylisted_hits > 0)) | length), + rows: [.[] | {repo: .pr.repo, number: .pr.number, disposition: .disposition, + blocked_by: .blocked_by, age_days: .age_days, + denylisted_hits: .denylisted_hits}] + } + ' "$DECISIONS" > "$TMP/prs.json" +else + echo '{"available": false, "source": null, "reason": "no pr-decisions.jsonl — run scripts/sweeps/estate-pr-automerge.sh first"}' > "$TMP/prs.json" +fi + +# ─── Open issues ───────────────────────────────────────────────────────── +: > "$TMP/issues.jsonl" +for org in "${ORGS[@]}"; do + for page in $(seq 1 10); do + n=$(gh api "search/issues?q=org:${org}+is:issue+is:open&per_page=100&page=${page}" \ + --jq '.items[] | {org:"'"$org"'", repo:(.repository_url | split("/") | last), n:.number, + title:.title, created:.created_at, author:.user.login, + labels:[.labels[].name], comments:.comments}' 2>/dev/null \ + | tee -a "$TMP/issues.jsonl" | wc -l) + [ "$n" -lt 100 ] && break + sleep 3 + done + sleep 2 +done +issue_count=$(wc -l < "$TMP/issues.jsonl") +log "open issues: ${issue_count}" + +# Duplicate-cause detection: normalise titles so that "3 pre-existing red +# checks" and "7 pre-existing red checks" collapse into one class. This is +# the number that matters for the sprawl question — 774 issues is a symptom, +# "N distinct causes" is the diagnosis. +jq -s ' + def norm: gsub("[0-9]+"; "N") | gsub("`[^`]*`"; "X"); + { + available: true, + total: length, + truncated: (. >= '"$MAX_ISSUES"'), + by_org: (group_by(.org) | map({key: .[0].org, value: length}) | from_entries), + by_author: (group_by(.author) | map({key: (.[0].author // "unknown"), value: length}) | sort_by(-.value) | .[0:6] | from_entries), + by_label: ([.[].labels[]] | sort | group_by(.) | map({key: (.[0] // "none"), value: length}) | sort_by(-.value) | .[0:12] | from_entries), + by_month: (group_by(.created[0:7]) | map({key: .[0].created[0:7], value: length}) | from_entries), + distinct_causes: (map(.title | norm) | unique | length), + top_causes: ([.[].title | norm] | group_by(.) | map({cause: .[0], count: length}) + | sort_by(-.count) | .[0:12]), + never_commented: (map(select(.comments == 0)) | length), + by_repo_top: (group_by(.repo) | map({key: .[0].repo, value: length}) + | sort_by(-.value) | .[0:12]) + } +' "$TMP/issues.jsonl" > "$TMP/issues.json" + +# ─── Test surface and suite outcomes (--checks) ────────────────────────── +# There is no test/bench data store in this estate: verisimdb-data/scans holds +# panic-attack security scans, not suite results. So "which repositories have +# no test surface at all" and "whose suites are failing" are measured from the +# only source that actually knows — the repositories' own workflow definitions +# and their runs on the default branch. +# +# It is a proxy and it says so in the artifact: a workflow counts as a test +# surface when its name or path matches (test|spec|ci|check|conformance|gate| +# audit|verify). Two API calls per active repository. +if [ "$CHECKS" -eq 1 ]; then + active_repos=$(jq -s '[.[] | select(.archived == false and .fork == false and .disabled == false)] | length' "$TMP/repos.jsonl") + log "checks: test surface + default-branch outcomes across ${active_repos} repos (2 calls each)…" + : > "$TMP/checks.jsonl" + i=0 + while IFS= read -r row; do + org=$(jq -r '.org' <<<"$row"); name=$(jq -r '.name' <<<"$row") + branch=$(jq -r '.default_branch // "main"' <<<"$row") + full="${org}/${name}" + i=$((i + 1)) + [ $((i % 25)) -eq 0 ] && log " … ${i}/${active_repos}" + + # A failed call is NOT an absence of tests. If either call fails the + # repository is recorded as unmeasured and excluded from the counts — + # otherwise a rate limit or a 409 on an empty repo would report as + # "this project has no test surface", which is a lie in the direction + # that matters. + measured=true + if wf_raw=$(timeout 45 gh api "repos/${full}/actions/workflows?per_page=100" 2>/dev/null); then + wf=$(jq -c '[.workflows[]? | {name:(.name // ""), path:(.path // ""), state:(.state // "active")}]' <<<"$wf_raw" 2>/dev/null) || wf='[]' + else + wf='[]'; measured=false + fi + if runs_raw=$(timeout 45 gh api "repos/${full}/actions/runs?per_page=20&branch=${branch}" 2>/dev/null); then + runs=$(jq -c '[.workflow_runs[]? | {name:(.name // ""), path:(.path // ""), head_branch:(.head_branch // ""), + status:(.status // ""), conclusion:(.conclusion // null), created:(.created_at // null), + updated:(.updated_at // null)}]' <<<"$runs_raw" 2>/dev/null) || runs='[]' + else + runs='[]'; measured=false + fi + [ -n "$wf" ] || wf='[]' + [ -n "$runs" ] || runs='[]' + + jq -cn --arg repo "$full" --arg branch "$branch" --argjson measured "$measured" --argjson wf "$wf" --argjson runs "$runs" ' + def is_test($w): (($w.name // "") | test("(test|spec|ci|check|conformance|gate|audit|verify)"; "i")) + or (($w.path // "") | test("(test|spec|ci|check|conformance|gate|audit|verify)"; "i")); + ([ $wf[] | select((.state // "active") == "active") | select(is_test(.)) ]) as $surface + # Explicitly newest-first by creation time: run ordering in the API + # response is not something to rely on when the answer is "this repo + # is red". (No apostrophes in here: this jq program lives inside a + # single-quoted shell string.) + | ([ $runs[] | select(.head_branch == $branch) | select(.status == "completed") | select(is_test(.)) ] + | sort_by(.created) | reverse) as $done + | { + repo: $repo, + default_branch: $branch, + measured: $measured, + workflows: ($wf | length), + test_surface: ($surface | length), + test_workflow: ($done[0].name // null), + test_conclusion: ($done[0].conclusion // null), + test_updated: ($done[0].updated // null) + }' >> "$TMP/checks.jsonl" + done < <(jq -c 'select(.archived == false and .fork == false and .disabled == false)' "$TMP/repos.jsonl") + log "checks: done" +fi + +# ─── Signals ───────────────────────────────────────────────────────────── +# Three sub-blocks, each with its own provenance. They are never merged into +# one "health score": a security finding and a failing suite are different +# facts about different surfaces. +weak='{"available": false, "reason": "no scan store passed via --scans; nothing was measured, so nothing is reported"}' +if [ -n "$SCANS_DIR" ] && [ -d "$SCANS_DIR" ]; then + weak=$(jq -s ' + { + available: true, + source: "panic-attack scan store (verisimdb-data/scans)", + repos_scanned: length, + repositories_with_weak_points: ([.[] | select(((.weak_points // []) | length) > 0)] | length), + weak_point_classes: ([.[].weak_points[]?.category] | map(select(. != null)) | sort | group_by(.) + | map({key: (.[0] // "uncategorised"), value: length}) | sort_by(-.value) | .[0:20] | from_entries) + } + ' "$SCANS_DIR"/*.json 2>/dev/null) || \ + weak='{"available": false, "reason": "scan store present but unreadable"}' + [ -n "$weak" ] || weak='{"available": false, "reason": "scan store present but unreadable"}' +fi + +tests='{"available": false, "reason": "no --checks run; test surface and suite outcomes were not measured"}' +if [ "$CHECKS" -eq 1 ] && [ -s "$TMP/checks.jsonl" ]; then + tests=$(jq -s ' + { + available: true, + source: "github actions api — active workflow definitions + runs on the default branch", + definition: "a workflow is a test surface when its name or path matches (test|spec|ci|check|conformance|gate|audit|verify)", + repos_measured: (map(select(.measured == true)) | length), + repos_unmeasured: (map(select(.measured != true)) | length), + with_test_surface: (map(select(.measured == true and .test_surface > 0)) | length), + coverage_empties: (map(select(.measured == true and .test_surface == 0)) | length), + coverage_empty_repos: ([.[] | select(.measured == true and .test_surface == 0) | .repo] | sort | .[0:60]), + failing: (map(select(.measured == true and .test_conclusion == "failure")) | length), + failing_repos: ([.[] | select(.measured == true and .test_conclusion == "failure") + | {repo, workflow: .test_workflow, at: .test_updated}] | sort_by(.repo) | .[0:60]), + never_concluded: (map(select(.measured == true and .test_surface > 0 and .test_conclusion == null)) | length), + last_run: ([.[].test_updated] | map(select(. != null)) | max // null) + } + ' "$TMP/checks.jsonl") +fi + +# Benches have no producer. The threshold block still evaluates them — as +# unavailable, because a threshold cannot be breached by data nobody collected. +benches='{"available": false, "reason": "no bench-result producer anywhere in the estate — no store, no workflow artefact contract", "expected": {"per_repo_file": "benches.json", "fields": ["repo", "bench", "value_ns", "limit_ns", "measured_at"]}}' + +signals=$(jq -cn --argjson weak "$weak" --argjson tests "$tests" --argjson benches "$benches" ' + { + available: ($weak.available or $tests.available), + security_weak_points: $weak, + tests: $tests, + benches: $benches + }') + +# ─── Paging: every policy threshold against the measured value ─────────── +# The dashboard reads this block instead of re-implementing the thresholds. +# `unavailable` for a metric with no source, never a silent "ok". +paging=$(jq -cn \ + --argjson thr "$(jq '.stats_thresholds' "$POLICY")" \ + --argjson prs "$(cat "$TMP/prs.json")" \ + --argjson issues "$(cat "$TMP/issues.json")" \ + --argjson signals "$signals" ' + def lvl($v; $w; $c): + if $v == null then "unavailable" + elif $v >= $c then "critical" + elif $v >= $w then "warn" + else "ok" end; + def row($v; $w; $c): {value: $v, warn: $w, critical: $c, level: lvl($v; $w; $c)}; + { + open_issue_count: row($issues.total; $thr.open_issue_count_warn; $thr.open_issue_count_critical), + unmerged_pr_count: row($prs.total; $thr.unmerged_pr_count_warn; $thr.unmerged_pr_count_critical), + unmerged_pr_age_days: row($prs.oldest_age_days; $thr.unmerged_pr_age_days_warn; $thr.unmerged_pr_age_days_critical), + failing_tests: row($signals.tests.failing; $thr.failing_tests_warn; $thr.failing_tests_critical), + test_coverage_empties: row($signals.tests.coverage_empties; $thr.test_coverage_empties_warn; $thr.test_coverage_empties_critical), + bench_coverage_empties: row(null; $thr.bench_coverage_empties_warn; $thr.bench_coverage_empties_critical), + bench_over_limit_ratio: row(null; $thr.bench_over_limit_ratio_warn; $thr.bench_over_limit_ratio_critical) + }') + +# ─── Assemble ──────────────────────────────────────────────────────────── +jq -n \ + --arg version "$(jq -r '.stats_output.schema_version' "$POLICY")" \ + --arg policy "$(jq -r '.version' "$POLICY")" \ + --arg run_at "$RUN_AT" \ + --argjson thresholds "$(jq '.stats_thresholds' "$POLICY")" \ + --argjson repos "$(cat "$TMP/absences.json")" \ + --argjson prs "$(cat "$TMP/prs.json")" \ + --argjson issues "$(cat "$TMP/issues.json")" \ + --argjson signals "$signals" \ + --argjson paging "$paging" \ + '{ + schema_version: $version, + policy_version: $policy, + generated_at: $run_at, + producer: "hypatia/scripts/sweeps/estate-stats.sh", + thresholds: $thresholds, + repositories: $repos, + pull_requests: $prs, + issues: $issues, + signals: $signals, + paging: $paging + }' > "$OUT" + +# ─── Human-readable companion ──────────────────────────────────────────── +{ + echo "= Estate statistics" + echo + echo "Generated ${RUN_AT} by \`estate-stats.sh\` (policy ${policy_version})." + echo + echo "== Pull requests" + jq -r 'if .available then + "- Open PRs: \(.total)\n- By disposition: \(.by_disposition | to_entries | map("\(.key)=\(.value)") | join(", "))\n- Carrying a denylisted pin: \(.poison_carrying)\n- Oldest: \(.oldest_age_days) days; median: \(.median_age_days) days" + else "- unavailable: \(.reason)" end' "$TMP/prs.json" + echo + echo "== Issues" + jq -r '"- Open: \(.total)\n- Distinct causes after normalisation: \(.distinct_causes)\n- Never commented on: \(.never_commented)\n- Top cause: \((.top_causes[0] // {cause:"-",count:0}) | "\(.count)× \(.cause)")"' "$TMP/issues.json" + echo + echo "== Repository absences" + jq -r '"- Repositories: \(.total) (\(.active) active, \(.archived) archived)\n- Empty description: \(.empty_description.count)\n- Fewer than \(.topics_below_min.min) topics: \(.topics_below_min.count)"' "$TMP/absences.json" + echo + echo "== Test surface" + jq -r 'if .available then + "- Repositories measured: \(.repos_measured) (unmeasured: \(.repos_unmeasured))\n- With a test surface: \(.with_test_surface)\n- No test surface at all: \(.coverage_empties)\n- Suites failing on the default branch: \(.failing)\n- Never concluded a test run: \(.never_concluded)" + else "- unavailable: \(.reason)" end' <<<"$tests" + echo + echo "== Benches" + jq -r 'if .available then "- measured" else "- unavailable: \(.reason)" end' <<<"$benches" + echo + echo "== Security weak points" + jq -r 'if .available then "- Repositories in the scan store: \(.repos_scanned)\n- With weak points: \(.repositories_with_weak_points)" else "- unavailable: \(.reason)" end' <<<"$weak" + echo + echo "== Thresholds (paging)" + jq -r 'to_entries[] | "- \(.key): \(.value.level)\(if .value.value != null then " (\(.value.value), warn \(.value.warn) / critical \(.value.critical))" else "" end)"' <<<"$paging" +} > "${OUT%.json}.adoc" 2>/dev/null || true + +log "" +log "─── estate stats ──────────────────────────────────────────────────" +jq -r '" repos : \(.repositories.total) (\(.repositories.active) active)\n empty descr. : \(.repositories.empty_description.count)\n <\(.repositories.topics_below_min.min) topics : \(.repositories.topics_below_min.count)\n open PRs : \(if .pull_requests.available then .pull_requests.total else "unavailable" end)\n open issues : \(if .issues.available then .issues.total else "unavailable" end)\n distinct causes : \(if .issues.available then .issues.distinct_causes else "unavailable" end)\n test surface : \(if .signals.tests.available then "\(.signals.tests.with_test_surface) with / \(.signals.tests.coverage_empties) without / \(.signals.tests.failing) failing" else "unavailable" end)\n benches : \(if .signals.benches.available then "measured" else "unavailable (no producer)" end)"' "$OUT" >&2 +log " paging : $(jq -r '[.paging[] | select(.level != "ok") | .level] | group_by(.) | map("\(length) \(.[0])") | join(", ")' "$OUT" | sed 's/^$/all ok/')" +log " artifact : ${OUT}" +log " human summary : ${OUT%.json}.adoc" + +exit 0 diff --git a/test/rules/pin_integrity_test.exs b/test/rules/pin_integrity_test.exs new file mode 100644 index 00000000..258eb3d3 --- /dev/null +++ b/test/rules/pin_integrity_test.exs @@ -0,0 +1,393 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Rules.PinIntegrityTest do + use ExUnit.Case, async: true + + alias Hypatia.Rules.PinIntegrity, as: PI + + # The two anchors of the September 2026 incident, spelled once. + @poison "1c5b675653bb5c22dbe9b12b556ec555138e09fd" + @good "b96794f015dfd88f77b49b1c93e0fa7110f94c63" + + # The shape `pr-automerge-policy.json` carries for PIN-001, trimmed to the + # fields the rules read. + @policy %{ + "pin_denylist" => [ + %{ + "id" => "PIN-001", + "action" => "github/codeql-action", + "severity" => "critical", + "blocked_versions" => ["4.38.1"], + "blocked_shas" => [@poison], + "blocked_refs" => ["v4.38.1"], + "reason" => + "GitHub rejects this commit at workflow start-up: `startup_failure`, " <> + "zero jobs, no logs, wherever it is used.", + "known_good_version" => "4.38.0", + "known_good_sha" => @good + }, + %{ + "id" => "PIN-002", + "action" => "dtolnay/rust-toolchain", + "severity" => "high", + "blocked_versions" => [], + "blocked_shas" => [], + "blocked_refs" => [] + } + ], + "moving_refs_never_lockable" => [ + "stable", + "beta", + "nightly", + "latest", + "main", + "master", + "HEAD" + ] + } + + @poisoned_file """ + name: codeql + jobs: + analyze: + steps: + - uses: actions/checkout@v7.0.1 + - uses: github/codeql-action/init@#{@poison} # v3 + - uses: github/codeql-action/analyze@#{@poison} # v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100) + """ + + describe "pin_sites/1" do + test "parses an indented, dash-led uses line and keeps the trailing comment" do + content = " - uses: github/codeql-action/init@#{@poison} # v4.38.0\n" + + assert [site] = PI.pin_sites(content) + assert site.line == 1 + assert site.action == "github/codeql-action/init" + assert site.action_base == "github/codeql-action" + assert site.ref == @poison + assert site.comment == "v4.38.0" + end + + test "ignores lines that are not uses pins" do + content = "name: ci\non: push\njobs:\n build:\n steps:\n - run: echo hi\n" + + assert PI.pin_sites(content) == [] + end + + test "a pin with no comment has an empty comment, not a missing one" do + assert [site] = PI.pin_sites("- uses: actions/checkout@v7.0.1\n") + assert site.comment == "" + end + end + + describe "action_base/1" do + test "keys on the release unit, not the sub-action" do + assert PI.action_base("github/codeql-action/init") == "github/codeql-action" + assert PI.action_base("actions/checkout") == "actions/checkout" + assert PI.action_base("not-a-path") == "not-a-path" + end + end + + describe "denylisted?/3" do + test "matches the SHA, the tag and the bare version — all three spellings" do + assert %{"id" => "PIN-001"} = PI.denylisted?(@policy, "github/codeql-action/init", @poison) + assert %{"id" => "PIN-001"} = PI.denylisted?(@policy, "github/codeql-action/analyze", "v4.38.1") + assert %{"id" => "PIN-001"} = PI.denylisted?(@policy, "github/codeql-action/init", "4.38.1") + end + + test "does not match a good ref, another action, or another repo's entry" do + refute PI.denylisted?(@policy, "github/codeql-action/init", @good) + assert PI.denylisted?(@policy, "dtolnay/rust-toolchain", "stable") == nil + assert PI.denylisted?(@policy, "actions/checkout", "v4.38.1") == nil + end + end + + describe "pi001_denylisted_pin/2 — a poisoned pin wearing a good label" do + test "fires on the SHA regardless of what the comment claims" do + content = """ + jobs: + analyze: + steps: + - uses: github/codeql-action/init@#{@poison} # v4.38.0 + """ + + assert [finding] = + PI.pi001_denylisted_pin(content, @policy, path: ".github/workflows/codeql.yml") + + assert finding.rule_id == "PI001" + assert finding.severity == :critical + assert finding.line == 4 + assert finding.path == ".github/workflows/codeql.yml" + assert finding.detail.denylist_id == "PIN-001" + assert finding.detail.known_good_sha == @good + assert finding.detail.repair == "substitution" + assert finding.description =~ "startup_failure" + end + + test "silent on a good pin and on an action the policy does not deny" do + content = """ + jobs: + analyze: + steps: + - uses: github/codeql-action/init@#{@good} # v4.38.0 + - uses: dtolnay/rust-toolchain@stable + """ + + assert PI.pi001_denylisted_pin(content, @policy) == [] + end + end + + describe "pi002_mislabelled_pin/2 — the comment against the resolved ref" do + test "the rollback that relabelled without re-pinning" do + poison = @poison + content = "- uses: github/codeql-action/init@#{@poison} # v4.38.0\n" + resolution = %{poison => "4.38.1"} + + assert [finding] = PI.pi002_mislabelled_pin(content, resolution) + assert finding.rule_id == "PI002" + assert finding.severity == :high + assert finding.detail.claimed == "4.38.0" + assert finding.detail.resolved == "4.38.1" + end + + test "silent when the ref cannot be resolved — an unjudgeable claim is not a finding" do + content = "- uses: github/codeql-action/init@#{@poison} # v4.38.0\n" + + assert PI.pi002_mislabelled_pin(content, %{}) == [] + end + + test "a coarse claim that does not contradict the resolved version is silent" do + ref = "aaaa1111bbbb2222cccc3333dddd4444eeee5555" + content = "- uses: actions/checkout@#{ref} # v3\n" + + assert PI.pi002_mislabelled_pin(content, %{ref => "3.1.0"}) == [] + end + + test "dictask's mislabel: a 4.38.1 pin annotated v3" do + poison = @poison + content = "- uses: github/codeql-action/init@#{@poison} # v3\n" + + assert [finding] = PI.pi002_mislabelled_pin(content, %{poison => "4.38.1"}) + assert finding.detail.claimed == "3" + assert finding.detail.resolved == "4.38.1" + end + + test "a comment with no version claim at all is not a finding" do + poison = @poison + content = "- uses: github/codeql-action/init@#{@poison} # do not move\n" + + assert PI.pi002_mislabelled_pin(content, %{poison => "4.38.1"}) == [] + end + end + + describe "claimed_version/1" do + test "reads the estate's real comment shapes" do + assert PI.claimed_version("v4.38.0") == "4.38.0" + assert PI.claimed_version("v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)") == "4.38.0" + assert PI.claimed_version("v3") == "3" + assert PI.claimed_version("Pinned to v1.2.3 — do not move") == "1.2.3" + end + + test "makes no claim out of prose" do + assert PI.claimed_version("do not move") == nil + assert PI.claimed_version("2 jobs") == nil + assert PI.claimed_version("") == nil + assert PI.claimed_version(nil) == nil + end + end + + describe "relabel/2 — byte-for-byte with relabel_comment in estate-pin-integrity.sh" do + test "relabels a comment that leads with a version claim" do + # The comment arrives from pin_sites/1 without its leading `#`. + assert PI.relabel("v3", "4.38.0") == "# v4.38.0" + assert PI.relabel("# v3", "4.38.0") == "# v4.38.0" + end + + test "leaves a comment that already carries the good version byte-identical" do + comment = "# v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)" + + assert PI.relabel(comment, "4.38.0") == comment + end + + test "leaves prose alone, whatever version it mentions mid-sentence" do + assert PI.relabel("# Pinned to v1.2.3 — do not move", "4.38.0") == + "# Pinned to v1.2.3 — do not move" + end + + test "empty stays empty, and a nil version is a no-op" do + assert PI.relabel("", "4.38.0") == "" + assert PI.relabel("# v3", nil) == "# v3" + end + end + + describe "substitute_denylisted_pins/2 — the whole of the mechanical repair" do + test "rewrites every denylisted site and relabels only leading version claims" do + assert {:ok, new_content, excisions} = + PI.substitute_denylisted_pins(@poisoned_file, @policy) + + assert length(excisions) == 2 + refute new_content =~ "1c5b6756" + assert length(Regex.scan(~r/b96794f0/, new_content)) == 2 + assert new_content =~ "codeql-action/init@#{@good}" + assert new_content =~ "codeql-action/analyze@#{@good}" + + [first, second] = excisions + assert first.action == "github/codeql-action/init" + assert first.from_ref == @poison + assert first.to_ref == @good + assert first.to_version == "4.38.0" + assert String.trim(first.comment_after) == "v4.38.0" + + # The rollback comment is a sentence, not a label: it must survive + # byte-identical, and the `# v3` mislabel must not survive at all. + assert second.comment_after == " v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)" + assert new_content =~ "# v4.38.0 (4.38.1 blocked estate-wide; nexia-list#100)" + refute new_content =~ "# v3" + end + + test "a file with no denylisted site is returned untouched" do + clean = "- uses: actions/checkout@v7.0.1\n" + + assert {:ok, ^clean, []} = PI.substitute_denylisted_pins(clean, @policy) + end + + test "refuses rather than guesses when the policy carries no known-good SHA" do + policy = %{ + "pin_denylist" => [ + %{ + "id" => "PIN-001", + "action" => "github/codeql-action", + "severity" => "critical", + "blocked_shas" => [@poison] + } + ] + } + + content = "- uses: github/codeql-action/init@#{@poison} # v3\n" + + assert {:error, reason} = PI.substitute_denylisted_pins(content, policy) + assert reason =~ "no known_good_sha" + end + end + + describe "pin_only_edit?/3 — the MGX-001 proof obligation" do + test "a token substitution on the same action across both sides" do + removed = [" - uses: github/codeql-action/init@#{@poison} # v4.38.1"] + added = [" - uses: github/codeql-action/init@#{@good} # v4.38.0"] + + assert PI.pin_only_edit?(removed, added, @policy) + end + + test "false when any changed line is not a uses pin" do + removed = [" - uses: github/codeql-action/init@#{@poison}"] + added = [" - uses: github/codeql-action/init@#{@good}", " permissions: {}"] + + refute PI.pin_only_edit?(removed, added, @policy) + end + + test "false when one action is swapped for another" do + removed = [" - uses: actions/checkout@v7.0.1"] + added = [" - uses: actions/setup-node@v7.0.1"] + + refute PI.pin_only_edit?(removed, added, @policy) + end + + test "false when the substitution moves onto the denylist" do + removed = [" - uses: github/codeql-action/init@#{@good}"] + added = [" - uses: github/codeql-action/init@#{@poison}"] + + refute PI.pin_only_edit?(removed, added, @policy) + end + + test "false when nothing actually changed" do + line = " - uses: actions/checkout@v7.0.1" + + refute PI.pin_only_edit?([line], [line], @policy) + end + + test "false when either side is empty" do + refute PI.pin_only_edit?([], [" - uses: actions/checkout@v7.0.1"], @policy) + refute PI.pin_only_edit?([" - uses: actions/checkout@v7.0.1"], [], @policy) + end + end + + describe "pi003_locked_moving_ref/2" do + test "a moving ref in a lockfile is a finding with a structural repair" do + lock = """ + '.github/workflows/security-policy.yml': + - 'dtolnay/rust-toolchain@stable' + - 'actions/checkout@v7.0.1' + """ + + assert [finding] = PI.pi003_locked_moving_ref(lock, @policy) + assert finding.rule_id == "PI003" + assert finding.severity == :high + assert finding.line == 2 + assert finding.detail.ref == "stable" + assert finding.detail.action == "dtolnay/rust-toolchain" + assert finding.detail.repair == "unpin_moving_ref" + end + + test "a non-moving ref is never a PI003" do + lock = " - 'actions/checkout@v7.0.1'\n" + + assert PI.pi003_locked_moving_ref(lock, @policy) == [] + end + end + + describe "pi004_lock_divergence/3" do + test "workflow and lockfile disagreeing about the same action" do + workflow = "- uses: github/codeql-action/init@#{@poison} # v4.38.1\n" + + lock = """ + '.github/workflows/codeql.yml': + - 'github/codeql-action@#{@good}' + """ + + assert [finding] = PI.pi004_lock_divergence(workflow, lock) + assert finding.rule_id == "PI004" + assert finding.severity == :medium + assert finding.detail.workflow_ref == @poison + assert finding.detail.lock_refs == [@good] + assert finding.detail.repair == "reconcile_lock_and_workflow" + end + + test "agreement is silence" do + workflow = "- uses: github/codeql-action/init@#{@good}\n" + lock = " - 'github/codeql-action@#{@good}'\n" + + assert PI.pi004_lock_divergence(workflow, lock) == [] + end + + test "an action the lockfile does not mention is not a divergence" do + workflow = "- uses: actions/checkout@v7.0.1\n" + lock = " - 'github/codeql-action@#{@good}'\n" + + assert PI.pi004_lock_divergence(workflow, lock) == [] + end + end + + describe "pi005_stale_pin/2 — advisory, never an action" do + test "an old pin with a versioned comment, reported as information" do + content = "- uses: actions/checkout@v4.1.7 # v4.1.7\n" + + assert [finding] = PI.pi005_stale_pin(content, %{"actions/checkout" => "4.2.0"}) + assert finding.rule_id == "PI005" + assert finding.severity == :info + assert finding.detail.latest == "4.2.0" + assert finding.detail.repair == "advisory" + end + + test "silent when the comment makes no version claim" do + content = "- uses: actions/checkout@v4.1.7 # pinned deliberately\n" + + assert PI.pi005_stale_pin(content, %{"actions/checkout" => "4.2.0"}) == [] + end + + test "silent when there is no upstream information" do + content = "- uses: actions/checkout@v4.1.7 # v4.1.7\n" + + assert PI.pi005_stale_pin(content, %{}) == [] + end + end +end diff --git a/test/rules/pr_automerge_test.exs b/test/rules/pr_automerge_test.exs new file mode 100644 index 00000000..645b7b2d --- /dev/null +++ b/test/rules/pr_automerge_test.exs @@ -0,0 +1,358 @@ +# SPDX-License-Identifier: MPL-2.0 + +defmodule Hypatia.Rules.PrAutomergeTest do + use ExUnit.Case, async: true + + alias Hypatia.Rules.PrAutomerge, as: PA + + # The two anchors of the September 2026 incident. + @poison "1c5b675653bb5c22dbe9b12b556ec555138e09fd" + @good "b96794f015dfd88f77b49b1c93e0fa7110f94c63" + + @policy %{ + "pin_denylist" => [ + %{ + "id" => "PIN-001", + "action" => "github/codeql-action", + "severity" => "critical", + "blocked_versions" => ["4.38.1"], + "blocked_shas" => [@poison], + "reason" => "GitHub rejects this commit at workflow start-up." + } + ], + "dependency_bots" => ["dependabot[bot]", "renovate[bot]"], + "lockfile_names" => ["Cargo.lock", "mix.lock", "package-lock.json"] + } + + # PR records arrive from Jason as string-keyed maps, which is the shape the + # tests use on purpose: atom-only reads are how an archived repo silently + # becomes an active one. + defp pr(files, opts \\ []) do + %{ + "author" => Keyword.get(opts, :author, "dependabot[bot]"), + "files" => files, + "body" => Keyword.get(opts, :body, ""), + "version_resolution" => Keyword.get(opts, :resolution, %{}), + "repo_archived" => Keyword.get(opts, :archived, false), + "repo" => "hyperpolymath/example", + "number" => 42, + "head_sha" => "0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d0d", + "base" => "main" + } + end + + defp file(name, patch), do: %{"filename" => name, "patch" => patch} + + defp checkout_patch(from, to) do + """ + @@ -10,1 +10,1 @@ + - - uses: actions/checkout@#{from} + + - uses: actions/checkout@#{to} + """ + end + + defp codeql_patch(from, to) do + """ + @@ -1,1 +1,1 @@ + - - uses: github/codeql-action/init@#{from} # v4.38.0 + + - uses: github/codeql-action/init@#{to} # v4.38.1 + """ + end + + describe "classify/2 — the unambiguous cases are armed" do + test "a lockfile-only bump is an auto-merge" do + decision = + PA.classify( + pr([file("mix.lock", "@@ -1,1 +1,1 @@\n- redix 1.2.0\n+ redix 1.3.0\n")]), + @policy + ) + + assert decision.disposition == :auto_merge + assert decision.blocked_by == "awaiting_required_checks" + assert decision.safety == "arm_auto" + assert decision.change_class == "bump" + assert decision.pool == "P2" + end + + test "a pin-only patch bump inside a workflow stays at object level" do + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v4.2.0"} => "4.2.0" + } + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v4.2.0"))], + resolution: resolution + ), + @policy + ) + + assert decision.disposition == :auto_merge + assert decision.change_level == "object" + assert decision.pin_only + + assert [delta] = decision.deltas + assert delta.status == :ok + assert delta.source == "tags" + refute delta.major? + end + + test "a body claim is enough when the refs cannot be resolved" do + body = "Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0.\n" + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v4.2.0"))], body: body), + @policy + ) + + assert decision.disposition == :auto_merge + assert [delta] = decision.deltas + assert delta.source == "body-claim" + end + end + + describe "classify/2 — PA002, majors are never armed" do + test "a version delta that crosses a major is flagged" do + resolution = %{ + {"actions/checkout", "v4.1.7"} => "4.1.7", + {"actions/checkout", "v7.0.1"} => "7.0.1" + } + + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v7.0.1"))], + resolution: resolution + ), + @policy + ) + + assert decision.disposition == :flag + assert decision.blocked_by == "major_version_delta" + assert decision.safety == "flag" + assert decision.major_delta + end + + test "an unresolvable delta is flagged rather than assumed" do + decision = + PA.classify( + pr([file(".github/workflows/ci.yml", checkout_patch("v4.1.7", "v4.2.0"))]), + @policy + ) + + assert decision.disposition == :flag + assert decision.blocked_by == "pin_delta_unresolvable" + assert decision.unresolved == 1 + end + end + + describe "classify/2 — the poison dispositions" do + test "a pin-only change onto the denylist is closed, not merged" do + decision = + PA.classify( + pr([file(".github/workflows/codeql.yml", codeql_patch(@good, @poison))]), + @policy + ) + + assert decision.disposition == :close_poison_only + assert decision.blocked_by == "introduces_denylisted_pin" + assert decision.safety == "flag" + assert decision.pool == "P1" + end + + test "poison alongside a wanted update is excised, then merged" do + patch = """ + @@ -1,4 +1,4 @@ + - - uses: github/codeql-action/init@#{@good} # v4.38.0 + + - uses: github/codeql-action/init@#{@poison} # v4.38.1 + - permissions: {} + + permissions: + + contents: read + """ + + decision = PA.classify(pr([file(".github/workflows/codeql.yml", patch)]), @policy) + + assert decision.disposition == :excise_poison_then_merge + assert decision.blocked_by == "introduces_denylisted_pin_alongside_wanted_updates" + refute decision.pin_only + end + + test "poison plus a major bump closes both reasons" do + resolution = %{ + {"github/codeql-action", @good} => "4.38.0", + {"github/codeql-action", @poison} => "4.38.1" + } + + decision = + PA.classify( + pr([file(".github/workflows/codeql.yml", codeql_patch(@good, @poison))], + resolution: resolution + ), + @policy + ) + + # resolved versions do not cross a major, so this stays a poison close… + assert decision.disposition == :close_poison_only + + # …while a genuine major alongside the poison takes the other branch. + major_resolution = %{ + {"github/codeql-action", "aaaa1111bbbb2222cccc3333dddd4444eeee5555"} => "3.28.0", + {"github/codeql-action", @poison} => "4.38.1" + } + + major_patch = codeql_patch("aaaa1111bbbb2222cccc3333dddd4444eeee5555", @poison) + + major_decision = + PA.classify( + pr([file(".github/workflows/codeql.yml", major_patch)], resolution: major_resolution), + @policy + ) + + assert major_decision.disposition == :close_poison_and_majors + assert major_decision.blocked_by == "introduces_denylisted_pin_and_major_bumps" + end + end + + describe "classify/2 — what must never be armed" do + test "an archived repository can never merge" do + decision = + PA.classify( + pr([file("mix.lock", "@@ -1,1 +1,1 @@\n- a 1.0.0\n+ a 1.0.1\n")], archived: true), + @policy + ) + + assert decision.disposition == :close_archived_repo + assert decision.blocked_by == "repository_is_archived_cannot_merge" + end + + test "a human PR keeps its human reviewer" do + decision = + PA.classify( + pr([file("mix.lock", "@@ -1,1 +1,1 @@\n- a 1.0.0\n+ a 1.0.1\n")], + author: "hyperpolymath" + ), + @policy + ) + + assert decision.disposition == :flag + assert decision.blocked_by == "not_a_dependency_bot" + end + + test "a licence touch routes to the owner, never the robot" do + patch = """ + @@ -1,1 +1,2 @@ + MPL-2.0 + +SPDX-License-Identifier: MPL-2.0 + """ + + decision = PA.classify(pr([file("LICENSE", patch)]), @policy) + + assert decision.disposition == :flag + assert decision.blocked_by == "touches_licence_requires_owner_review" + assert decision.licence_touch + end + + test "a dependency manifest without a lockfile is flagged" do + patch = """ + @@ -3,1 +3,1 @@ + - "left-pad": "1.0.0", + + "left-pad": "1.0.1", + """ + + decision = PA.classify(pr([file("package.json", patch)]), @policy) + + assert decision.disposition == :flag + assert decision.blocked_by == "dependency_manifest_not_lockfile" + end + end + + describe "poison_sites/2 and pin_lines_only?/1" do + test "the poisoned SHA is found on the added side only" do + patch = codeql_patch(@good, @poison) + sites = PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) + + assert [site] = sites + assert site.denylist_id == "PIN-001" + assert site.file == ".github/workflows/codeql.yml" + assert PA.pin_lines_only?(file(".github/workflows/codeql.yml", patch)) + end + + test "a patch that also edits a permissions block is not pin-only" do + patch = """ + @@ -1,4 +1,4 @@ + - - uses: github/codeql-action/init@#{@good} # v4.38.0 + + - uses: github/codeql-action/init@#{@poison} # v4.38.1 + permissions: {} + contents: read + """ + + refute PA.pin_lines_only?(file(".github/workflows/codeql.yml", patch)) + end + + test "a good pin is not a poison site" do + patch = codeql_patch(@poison, @good) + + assert PA.poison_sites([file(".github/workflows/codeql.yml", patch)], @policy) == [] + end + end + + describe "body_claims/1" do + test "the grouped and single-claim shapes are both read" do + body = """ + Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0. + Updates `github/codeql-action/init` from 4.38.0 to 4.38.1 + """ + + claims = PA.body_claims(body) + + assert Enum.any?(claims, &(&1.action == "actions/checkout" and &1.from == "4.1.7" and &1.to == "4.2.0")) + assert Enum.any?(claims, &(&1.action == "github/codeql-action/init" and &1.to == "4.38.1")) + end + + test "a dotted version is not truncated by the sentence's full stop" do + claims = PA.body_claims("Bumps [x](url) from 4.38.0 to 4.38.1.") + + assert [%{from: "4.38.0", to: "4.38.1"}] = claims + end + + test "prose without a claim yields nothing" do + assert PA.body_claims("This PR is fine, honestly.") == [] + end + end + + describe "decision_manifest/2" do + test "an armed decision has no vetoes" do + decision = + PA.classify( + pr([file("mix.lock", "@@ -1,1 +1,1 @@\n- a 1.0.0\n+ a 1.0.1\n")]), + @policy + ) + + manifest = PA.decision_manifest(decision, pr([])) + + assert manifest["safety"] == "arm_auto" + assert manifest["vetoes"] == [] + assert manifest["clamped_by"] == nil + assert manifest["pr"]["author"] == "dependabot[bot]" + assert manifest["pr"]["number"] == 42 + assert manifest["disposition"] == "auto_merge" + assert manifest["blocked_by"] == "awaiting_required_checks" + end + + test "a flagged decision carries a veto and is clamped" do + patch = """ + @@ -1,1 +1,2 @@ + MPL-2.0 + +SPDX-License-Identifier: MPL-2.0 + """ + + decision = PA.classify(pr([file("LICENSE", patch)]), @policy) + manifest = PA.decision_manifest(decision, pr([])) + + assert manifest["safety"] == "flag" + assert manifest["clamped_by"] == "veto" + assert [%{"reason" => "touches_licence_requires_owner_review"} | _] = manifest["vetoes"] + end + end +end From 3665cd226489375540191ce3334585491e0b3543 Mon Sep 17 00:00:00 2001 From: hypatia Date: Sat, 26 Sep 2026 14:13:08 +0000 Subject: [PATCH 2/6] feat(stats): measure the test surface; page every policy threshold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The signals section could only ever report security weak points, because the scan store holds panic-attack scans and nothing else. "Which repositories have no tests at all" and "whose suites are failing" — two of the four things the dashboard was asked to track — had no source and so reported `unavailable` forever. They were measurable all along, from the repositories' own workflow definitions and their runs on the default branch. * --checks: two API calls per active repository; a workflow counts as a test surface when its name or path matches (test|spec|ci|check| conformance|gate|audit|verify). Opt-in: ~9 minutes across 408 repos. * A failed call is recorded as unmeasured and excluded from the counts, so a rate limit can never read as "this project has no tests". * The newest completed run wins, sorted on created_at rather than trusting API ordering — this changed the answer for two repositories. * New top-level `paging` block: every threshold in stats_thresholds evaluated against the measured value, with `unavailable` for the metrics that have no producer (benches) instead of a silent `ok`. Measured on 2026-09-26: 408 repositories, 350 with a test surface, 58 with none, 120 whose most recent suite failed. Paging: 3 critical (open issues 887, failing tests, coverage empties), 2 warn (PR count, oldest PR age), 2 unavailable (the bench metrics). Also adds an `estate-rules` CI job: the two new rule test files and a structure gate over the four sweeps. Targeted rather than `mix test`, because the repository has known-red test families unrelated to these rules and a gate that is red on arrival gets ignored. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/tests.yml | 55 +++++++++++++++++++++ docs/operations/estate-automerge.adoc | 69 +++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 09c3a3a7..6e62f5b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,61 @@ concurrency: permissions: read-all jobs: + # --------------------------------------------------------------------------- + # Estate rule modules (PI001-PI005, PA001-PA006) and the sweeps that drive + # them. These tests exist because of the 2026-09-22 codeql-action rollback + # that relabelled a poisoned pin instead of replacing it: a version-string + # check saw nothing wrong, and the estate merged the poison back in. + # No network, no token: the rules are pure, and the sweeps are only + # syntax-checked here — their dry runs live in estate-rescan.yml. + # --------------------------------------------------------------------------- + estate-rules: + name: Estate rules and sweep structure + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@v7.0.1 + + - name: Setup Erlang/Elixir + uses: erlef/setup-beam@v1.24.1 + with: + otp-version: '27.0' + elixir-version: '1.17' + + - name: Restore Mix cache + uses: actions/cache@v6.1.0 + with: + path: | + deps + _build + key: ${{ runner.os }}-mix-${{ hashFiles('mix.lock') }} + + - name: Install dependencies + run: mix deps.get + + - name: Rule tests — pin integrity and PR automerge + # Targeted, not `mix test`: the repository has known-red test families + # that are unrelated to these rules, and a gate that is red on arrival + # teaches people to ignore it. + run: mix test test/rules/pin_integrity_test.exs test/rules/pr_automerge_test.exs + + - name: Sweep structure gate + run: | + set -euo pipefail + for s in estate-pin-integrity estate-pr-automerge estate-stats estate-absence-intake; do + bash -n "scripts/sweeps/${s}.sh" + echo "ok: ${s}.sh" + done + + - name: Policy is valid JSON and names its producer + run: | + set -euo pipefail + policy=".machine_readable/merge-orchestration/pr-automerge-policy.json" + jq -e '.version and .pin_denylist and .stats_thresholds and .stats_output.producer_path' "$policy" >/dev/null + echo "policy $(jq -r .version "$policy") ok" + e2e-elixir: name: E2E — Elixir Scanner Pipeline runs-on: ubuntu-latest diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index 53f3d631..ba947ef8 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -167,11 +167,16 @@ Turn unresolvable absences into issues exactly once. Emit the single estate-wide statistics artifact the private dashboard reads. * Flags: `--org` (repeatable), `--out FILE`, `--decisions FILE`, - `--scans DIR`, `--max-issues N`. -* Output: `estate-stats.json` plus a human-readable `.adoc` companion. + `--scans DIR`, `--checks`, `--max-repos N`, `--max-issues N`. +* Output: `estate-stats.json` plus a human-readable `.adoc` companion, and a + top-level `paging` block that evaluates every threshold in + `stats_thresholds` against the measured value. * Every section carries a `provenance` block; a metric that could not be measured is reported as `unavailable`, never as `0`. Numbers that were never fetched must not be able to look green. +* `--checks` adds two API calls per active repository and measures the test + surface and the suite outcome. This is the expensive flag: roughly nine + minutes across 408 repositories, so it is opt-in rather than default. == The dashboard contract @@ -184,9 +189,43 @@ with dated copies under the history directory recorded in `stats_output`. Sections: `repositories` (archived/empty-description/topics-low counts), `pull_requests` (counts by disposition, poisoned count, oldest/median age), -`issues` (open count, distinct causes, never-commented), `signals` -(test-coverage empties, failing tests, bench ratios, where measured). -Thresholds come from `stats_thresholds`; the artifact does not invent its own. +`issues` (open count, distinct causes, never-commented), `signals`, and +`paging`. + +`signals` has three sub-blocks that are never merged into one "health score", +because a security finding and a failing suite are different facts about +different surfaces: + +[cols="2,3,2", options="header"] +|=== +| Sub-block | Source | Measured today + +| `security_weak_points` +| panic-attack scan store (`verisimdb-data/scans`, via `--scans`) +| 541 repositories, 393 with weak points + +| `tests` +| GitHub Actions API (via `--checks`): active workflow definitions plus runs + on the default branch +| 408 repositories measured, 350 with a test surface, 58 with none, 120 whose + most recent suite failed + +| `benches` +| none — no producer exists in the estate +| unavailable, with the file/field contract it must satisfy +|=== + +The `tests` numbers are a proxy and the artifact says so: a workflow counts as +a test surface when its name or path matches +`(test|spec|ci|check|conformance|gate|audit|verify)`, and "failing" means the +most recent completed test-surface run on the default branch. A repository +whose calls failed is recorded as `unmeasured` and excluded from the counts — +a rate limit must never be able to read as "this project has no tests". + +`paging` is the block the dashboard should read first: each threshold from +`stats_thresholds` with its measured value and a `level` of `ok`, `warn`, +`critical` or `unavailable`. Thresholds come from the policy; the artifact does +not invent its own. == Operating procedure @@ -201,6 +240,26 @@ Thresholds come from `stats_thresholds`; the artifact does not invent its own. . `./scripts/sweeps/estate-stats.sh …` after the run; the artifact is the record of what changed. +== Not yet implemented + +Three things are specified in the policy but not yet built, listed here so +their absence is visible rather than discovered: + +* **ABS-003 (repository chrome)** — detection of a missing + `LICENSE`/`README.adoc`/`SECURITY.adoc` is specified with + `auto_fixable: seed_from_template`. It is not wired into the intake sweep + yet. Seeding needs a write token, and filing it as an absence would be + wrong (it is fixable, so by the policy's own rule it is fixed, not filed) — + which leaves "detect and report in the stats artifact" as the correct + handling, not yet written. +* **Bench results** — `bench_coverage_empties` and `bench_over_limit_ratio` + have thresholds and no producer: no store, no workflow artefact contract. + Their `expected` shape is recorded in `signals.benches`, so a producer can + be written against it. Until then they page as `unavailable`, which is not + the same as zero and must not be displayed as such. +* **Test/bench history** — the artifact is a snapshot. The consumer's + `history_dir` is where dated copies accumulate; nothing writes them yet. + == What this machinery deliberately will not do * Merge a major bump, a security advisory, a workflow add/remove, a From 2002413cae769fdbbf8a9bbb1b515d9a2512ac1d Mon Sep 17 00:00:00 2001 From: hypatia Date: Sat, 26 Sep 2026 14:19:35 +0000 Subject: [PATCH 3/6] docs: two causes of startup_failure, told apart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An agent-pushed pull request gets no pull_request workflow runs at all: every one fails at startup with zero jobs, including workflows the pull request never touches. That is the same signature as the codeql-action denylist failure, and the GitHub UI gives both the same explanation. Established by elimination on #862: 18 of 18 failed, and they still failed with an empty commit and with the workflow edits removed entirely. The same wall appears on MetaManifold-WebUI#72 and oikosbot#107, both agent-pushed and both merged on owner review. This matters because the whole point of the estate machinery is that a robot proposes pin repairs — and a robot's repair arrives wearing the costume of the defect it removes. The rule of thumb, written into the document: if a workflow the pull request does not touch fails at startup, the cause is not the pull request's contents. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- docs/operations/estate-automerge.adoc | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index ba947ef8..91d2be5f 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -240,6 +240,50 @@ not invent its own. . `./scripts/sweeps/estate-stats.sh …` after the run; the artifact is the record of what changed. +== Diagnosing startup_failure: two causes, one costume + +`startup_failure` with zero jobs is the most misleading signature in this +estate, because two unrelated causes produce it and GitHub shows the same +message for both: "This run likely failed because of a workflow file issue." + +[cols="1,3,3", options="header"] +|=== +| | Content — the denylist | Actor — a robot-pushed branch + +| Which runs fail +| only the workflows that use a denylisted pin +| every workflow triggered by the pull request, including files the pull + request never touches + +| Evidence +| hypatia PR #859 (human, no workflow edits): 4 of 19 failed, all + codeql-action users. Dependabot PR #861: 7 of 19. +| hypatia#862 (pushed by the estate's agent): 18 of 18, and it stayed 18 of 18 + with an empty commit and with the workflow edits removed. MetaManifold-WebUI#72 + and oikosbot#107 — both agent-pushed, both merged — show the same wall. + +| What to do +| substitute the pin; see `estate-pin-integrity.sh` +| nothing to fix in the branch: push it from a human account, or accept owner + review as the gate +|=== + +Rule of thumb: **if a workflow the pull request does not touch fails at +startup, the cause is not the pull request's contents.** + +Why this matters more than the mechanics: the entire point of this machinery is +that a robot proposes pin repairs, and a robot's repair arrives wearing the +exact costume of the defect it removes. An owner seeing eighteen red X's will +reasonably conclude the fix is broken. It is not — but nothing in the GitHub +UI will say so, which is why this page does. + +Consequence for gating: `pull_request` runs do not start for branches pushed +by the estate's automation, so required status checks never satisfy and such a +pull request sits `BLOCKED` until the owner overrides. Estate automation must +gate on checks it can actually run itself — the `dynamic`-event PR validation +runs do execute (`PR #72`, `PR #107`) — and must treat GitHub's own pull +request checks on its own pull requests as **unavailable, never as passing**. + == Not yet implemented Three things are specified in the policy but not yet built, listed here so From b1b792026b4d5812a9fe37970f242f30e485b51a Mon Sep 17 00:00:00 2001 From: hypatia Date: Sat, 26 Sep 2026 14:20:01 +0000 Subject: [PATCH 4/6] docs(status): commit the 2026-09-26 estate statistics artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The policy names docs/status/estate-stats.json as the producer path for the private farm dashboard; this is that file, generated by scripts/sweeps/estate-stats.sh with --checks, plus a README saying how to regenerate it and what the headline numbers mean. The numbers this run establishes, none of which were visible before: test surface 350 repositories with, 58 without, 120 whose most recent suite failed (408 measured; a failed API call is recorded as unmeasured and excluded, so a rate limit cannot read as "this project has no tests") paging 3 critical (open issues 887, failing suites, coverage empties), 2 warn (PR count 33, oldest PR 7 days), 2 unavailable (the bench metrics — no producer exists, and unavailable is not zero) Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- docs/status/README.adoc | 35 ++ docs/status/estate-stats.adoc | 43 ++ docs/status/estate-stats.json | 1109 +++++++++++++++++++++++++++++++++ 3 files changed, 1187 insertions(+) create mode 100644 docs/status/README.adoc create mode 100644 docs/status/estate-stats.adoc create mode 100644 docs/status/estate-stats.json diff --git a/docs/status/README.adoc b/docs/status/README.adoc new file mode 100644 index 00000000..d159da04 --- /dev/null +++ b/docs/status/README.adoc @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 += Estate statistics artifacts +:author: Jonathan D.A. Jewell +:date: 2026-09-26 + +`estate-stats.json` is the artifact the private farm dashboard consumes +(`stats_output` in `.machine_readable/merge-orchestration/pr-automerge-policy.json` +names this path as the producer path). `estate-stats.adoc` is its +human-readable companion. + +It is generated, not authored: + + ./scripts/sweeps/estate-stats.sh \ + --org hyperpolymath --org metadatastician \ + --out docs/status/estate-stats.json \ + --decisions .pr-automerge/pr-decisions.jsonl \ + --scans /path/to/verisimdb-data/scans \ + --checks + +`--checks` is the expensive flag (two API calls per active repository, about +nine minutes across the estate) and it is what measures the test surface and +suite outcomes. Without it those numbers are `unavailable`, which is +deliberate: an unmeasured metric must never be able to read as zero. + +The copy committed here is the 2026-09-26 run. Its headline numbers: + +* 416 repositories (413 active), 6 with no description, 43 below the 7-topic minimum +* 33 open pull requests; 25 carrying a denylisted pin; oldest 7 days +* 887 open issues, 843 distinct causes after normalising titles, 662 never commented on +* test surface: 350 repositories with, 58 without, 120 whose most recent suite failed +* benches: unavailable — no producer exists in the estate +* paging: 3 critical (open issues, failing suites, coverage empties), + 2 warn (PR count, oldest PR age), 2 unavailable (the bench metrics) + +The private farm should consume the JSON, never the prose. diff --git a/docs/status/estate-stats.adoc b/docs/status/estate-stats.adoc new file mode 100644 index 00000000..bcd49f19 --- /dev/null +++ b/docs/status/estate-stats.adoc @@ -0,0 +1,43 @@ += Estate statistics + +Generated 2026-09-26T14:03:46Z by `estate-stats.sh` (policy 1.0.0). + +== Pull requests +- Open PRs: 33 +- By disposition: auto_merge=5, close_archived_repo=1, close_poison_and_majors=4, close_poison_only=21, flag=2 +- Carrying a denylisted pin: 26 +- Oldest: 7 days; median: 4 days + +== Issues +- Open: 887 +- Distinct causes after normalisation: 843 +- Never commented on: 662 +- Top cause: 12× Security: Dependency vulnerabilities detected + +== Repository absences +- Repositories: 416 (413 active, 2 archived) +- Empty description: 6 +- Fewer than 7 topics: 43 + +== Test surface +- Repositories measured: 408 (unmeasured: 0) +- With a test surface: 350 +- No test surface at all: 58 +- Suites failing on the default branch: 120 +- Never concluded a test run: 17 + +== Benches +- unavailable: no bench-result producer anywhere in the estate — no store, no workflow artefact contract + +== Security weak points +- Repositories in the scan store: 541 +- With weak points: 393 + +== Thresholds (paging) +- open_issue_count: critical (887, warn 250 / critical 500) +- unmerged_pr_count: warn (33, warn 10 / critical 50) +- unmerged_pr_age_days: warn (7, warn 7 / critical 30) +- failing_tests: critical (120, warn 1 / critical 25) +- test_coverage_empties: critical (58, warn 11 / critical 47) +- bench_coverage_empties: unavailable +- bench_over_limit_ratio: unavailable diff --git a/docs/status/estate-stats.json b/docs/status/estate-stats.json new file mode 100644 index 00000000..33d4d57c --- /dev/null +++ b/docs/status/estate-stats.json @@ -0,0 +1,1109 @@ +{ + "schema_version": "1.0.0", + "policy_version": "1.0.0", + "generated_at": "2026-09-26T14:03:46Z", + "producer": "hypatia/scripts/sweeps/estate-stats.sh", + "thresholds": { + "comment": "Live thresholds for the private-farm dashboard. A metric crosses into 'recent/live tracking' when it exceeds these; below them it is reported as a headline number only.", + "failing_tests_warn": 1, + "failing_tests_critical": 25, + "test_coverage_empties_warn": 11, + "test_coverage_empties_critical": 47, + "bench_coverage_empties_warn": 11, + "bench_coverage_empties_critical": 47, + "bench_over_limit_ratio_warn": 1.5, + "bench_over_limit_ratio_critical": 3, + "unmerged_pr_age_days_warn": 7, + "unmerged_pr_age_days_critical": 30, + "unmerged_pr_count_warn": 10, + "unmerged_pr_count_critical": 50, + "open_issue_count_warn": 250, + "open_issue_count_critical": 500 + }, + "repositories": { + "total": 416, + "active": 413, + "archived": 2, + "empty_description": { + "count": 6, + "repos": [ + "hyperpolymath/first-post", + "hyperpolymath/gsbot", + "hyperpolymath/hotchocolabot", + "hyperpolymath/jtv-halting-islands-ct", + "hyperpolymath/JuliaForChildren.jl", + "hyperpolymath/stateful-artefacts" + ] + }, + "topics_below_min": { + "min": 7, + "count": 43, + "repos": [ + { + "repo": "hyperpolymath/.github", + "topics": 0 + }, + { + "repo": "hyperpolymath/EpistemicTypes.jl", + "topics": 0 + }, + { + "repo": "hyperpolymath/explore", + "topics": 0 + }, + { + "repo": "hyperpolymath/jtv-halting-islands-ct", + "topics": 0 + }, + { + "repo": "hyperpolymath/MetaManifold-WebUI", + "topics": 0 + }, + { + "repo": "hyperpolymath/Protoctist.jl", + "topics": 0 + }, + { + "repo": "metadatastician/authority-watch", + "topics": 0 + }, + { + "repo": "metadatastician/sim-public-relations", + "topics": 0 + }, + { + "repo": "metadatastician/harvard-dehallucinator", + "topics": 0 + }, + { + "repo": "metadatastician/.github", + "topics": 0 + }, + { + "repo": "metadatastician/stealth-glider-ecosystem", + "topics": 0 + }, + { + "repo": "metadatastician/x15-rocket-glider", + "topics": 0 + }, + { + "repo": "metadatastician/planer-1", + "topics": 0 + }, + { + "repo": "metadatastician/enterglide-ncc-137-k", + "topics": 0 + }, + { + "repo": "metadatastician/tardis-rule-disruption", + "topics": 0 + }, + { + "repo": "metadatastician/MaridIR.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridCore.jl", + "topics": 0 + }, + { + "repo": "metadatastician/ArangoDB.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridCodec.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridOpenAPI.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridRPC.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridGraphQL.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridTransport.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridStorage.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridControl.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridLive.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridCRDT.jl", + "topics": 0 + }, + { + "repo": "metadatastician/MaridRaft.jl", + "topics": 0 + }, + { + "repo": "metadatastician/marid-client", + "topics": 0 + }, + { + "repo": "metadatastician/marid-elements", + "topics": 0 + }, + { + "repo": "metadatastician/marid-react", + "topics": 0 + }, + { + "repo": "metadatastician/marid-vue", + "topics": 0 + }, + { + "repo": "metadatastician/marid-relationship-explorer", + "topics": 0 + }, + { + "repo": "metadatastician/knot-knot", + "topics": 0 + }, + { + "repo": "metadatastician/first-post", + "topics": 3 + }, + { + "repo": "hyperpolymath/veridical-simulation-core", + "topics": 5 + }, + { + "repo": "hyperpolymath/cargo-zigbuild", + "topics": 6 + }, + { + "repo": "hyperpolymath/first-post", + "topics": 6 + }, + { + "repo": "hyperpolymath/gsbot", + "topics": 6 + }, + { + "repo": "hyperpolymath/hotchocolabot", + "topics": 6 + }, + { + "repo": "hyperpolymath/lol", + "topics": 6 + }, + { + "repo": "hyperpolymath/stateful-artefacts", + "topics": 6 + }, + { + "repo": "metadatastician/large-language-michelangelo", + "topics": 6 + } + ] + } + }, + "pull_requests": { + "available": true, + "source": "pr-decisions.jsonl", + "total": 33, + "by_disposition": { + "auto_merge": 5, + "close_archived_repo": 1, + "close_poison_and_majors": 4, + "close_poison_only": 21, + "flag": 2 + }, + "by_blocked_by": { + "awaiting_required_checks": 5, + "dependency_manifest_not_lockfile": 1, + "introduces_denylisted_pin": 21, + "introduces_denylisted_pin_and_major_bumps": 4, + "major_version_delta": 1, + "repository_is_archived_cannot_merge": 1 + }, + "by_repo_archived": 0, + "oldest_age_days": 7, + "median_age_days": 4, + "poison_carrying": 26, + "rows": [ + { + "repo": "hyperpolymath/bgp-backbone-lab", + "number": 102, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/blue-screen-of-app", + "number": 85, + "disposition": "auto_merge", + "blocked_by": "awaiting_required_checks", + "age_days": 6, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/bofig", + "number": 204, + "disposition": "close_poison_and_majors", + "blocked_by": "introduces_denylisted_pin_and_major_bumps", + "age_days": 1, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/casket-ssg", + "number": 102, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/chimichanga", + "number": 100, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 1, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/claude-gecko-browser-extension", + "number": 110, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/claude-gecko-browser-extension", + "number": 109, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/Cliodynamics.jl", + "number": 72, + "disposition": "close_poison_and_majors", + "blocked_by": "introduces_denylisted_pin_and_major_bumps", + "age_days": 3, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/ensaid-spec", + "number": 37, + "disposition": "close_poison_and_majors", + "blocked_by": "introduces_denylisted_pin_and_major_bumps", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/git-reticulator", + "number": 115, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/git-reticulator", + "number": 114, + "disposition": "auto_merge", + "blocked_by": "awaiting_required_checks", + "age_days": 4, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/hyperpolymath.github.io", + "number": 42, + "disposition": "close_poison_and_majors", + "blocked_by": "introduces_denylisted_pin_and_major_bumps", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/ipv6-site-enforcer", + "number": 99, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/JuliaForChildren.jl", + "number": 25, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 3, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/lol", + "number": 15, + "disposition": "flag", + "blocked_by": "dependency_manifest_not_lockfile", + "age_days": 5, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/marches", + "number": 32, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 3, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/megadog", + "number": 71, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/network-outpost", + "number": 29, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/nexia-list", + "number": 108, + "disposition": "auto_merge", + "blocked_by": "awaiting_required_checks", + "age_days": 1, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/nextgen-languages", + "number": 162, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 0, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/oblibeny", + "number": 136, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 0, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/panic-attack", + "number": 200, + "disposition": "auto_merge", + "blocked_by": "awaiting_required_checks", + "age_days": 6, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/pow-the-game", + "number": 101, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/qubes-sdp", + "number": 78, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/reasonably-good-token-vault", + "number": 185, + "disposition": "auto_merge", + "blocked_by": "awaiting_required_checks", + "age_days": 4, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/self-destructing-git-garbage", + "number": 25, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/social-media-polygraph", + "number": 107, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 2 + }, + { + "repo": "hyperpolymath/tangle", + "number": 126, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 5, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/the-metadatastician", + "number": 47, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/verisimdb", + "number": 284, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 4, + "denylisted_hits": 1 + }, + { + "repo": "hyperpolymath/ViableSystems.jl", + "number": 27, + "disposition": "flag", + "blocked_by": "major_version_delta", + "age_days": 7, + "denylisted_hits": 0 + }, + { + "repo": "hyperpolymath/zerostep", + "number": 97, + "disposition": "close_poison_only", + "blocked_by": "introduces_denylisted_pin", + "age_days": 3, + "denylisted_hits": 1 + }, + { + "repo": "metadatastician/canonical-ums", + "number": 20, + "disposition": "close_archived_repo", + "blocked_by": "repository_is_archived_cannot_merge", + "age_days": 3, + "denylisted_hits": 2 + } + ] + }, + "issues": { + "available": true, + "total": 887, + "truncated": true, + "by_org": { + "hyperpolymath": 782, + "metadatastician": 105 + }, + "by_author": { + "hyperpolymath": 853, + "arena-ai-coding-agent[bot]": 20, + "github-actions[bot]": 13, + "nkar123412-hub": 1 + }, + "by_label": { + "enhancement": 166, + "cicd": 112, + "bug": 77, + "chore": 72, + "tech-debt": 69, + "documentation": 57, + "governance": 51, + "security": 43, + "proofs": 41, + "meta:roadmap": 37, + "automation": 36, + "scope:estate": 36 + }, + "by_month": { + "2026-03": 2, + "2026-05": 88, + "2026-06": 292, + "2026-07": 91, + "2026-08": 85, + "2026-09": 329 + }, + "distinct_causes": 843, + "top_causes": [ + { + "cause": "Security: Dependency vulnerabilities detected", + "count": 12 + }, + { + "cause": "Triage: N pre-existing red checks surfaced by the actions.lock cure (PR #N)", + "count": 12 + }, + { + "cause": "Repository description is empty", + "count": 6 + }, + { + "cause": "Machine-readable currency: deferred follow-ups (post-checkpoint #N)", + "count": 4 + }, + { + "cause": "Triage: N checks failing on fix/lock-coverage, newly visible after workflow startup was restored", + "count": 3 + }, + { + "cause": "Triage: N pre-existing red checks surfaced by the actions.lock cure (PR #N) — incl. a live ANML gate", + "count": 3 + }, + { + "cause": "Author tiered dev/maintainer/user docs + concept guides", + "count": 2 + }, + { + "cause": "Canonical parse-AST (expected.json) + expand conformance corpus", + "count": 2 + }, + { + "cause": "Crypto: no hashing, HMAC or RSA signing - blocks GitHub App auth from AffineScript (priority)", + "count": 2 + }, + { + "cause": "Governance parity: stamp standard .machine_readable + CONTRIBUTING/SECURITY/COC from standards", + "count": 2 + }, + { + "cause": "Member rollout: ANCHOR satellite+pin+conformance CI; enable upstream-pins", + "count": 2 + }, + { + "cause": "Priority test and benchmark backlog (Proven-Tests adoption)", + "count": 2 + } + ], + "never_commented": 662, + "by_repo_top": [ + { + "key": "standards", + "value": 192 + }, + { + "key": "affinescript", + "value": 56 + }, + { + "key": "hypatia", + "value": 53 + }, + { + "key": "paint-type", + "value": 30 + }, + { + "key": "valence-shell", + "value": 28 + }, + { + "key": "panoply", + "value": 24 + }, + { + "key": "marid", + "value": 23 + }, + { + "key": "echo-types", + "value": 22 + }, + { + "key": "boj-server", + "value": 17 + }, + { + "key": "nextgen-typing", + "value": 16 + }, + { + "key": "proven", + "value": 16 + }, + { + "key": "absolute-zero", + "value": 15 + } + ] + }, + "signals": { + "available": true, + "security_weak_points": { + "available": true, + "source": "panic-attack scan store (verisimdb-data/scans)", + "repos_scanned": 541, + "repositories_with_weak_points": 393, + "weak_point_classes": { + "MutationGap": 809, + "UnsafeCode": 772, + "CommandInjection": 718, + "PanicPath": 674, + "UnboundedAllocation": 460, + "InsecureProtocol": 382, + "UnsafeFFI": 350, + "DynamicCodeExecution": 233, + "ProofDrift": 231, + "UncheckedError": 221, + "PathTraversal": 177, + "InputBoundary": 119, + "HardcodedSecret": 93, + "UnsafeDeserialization": 32, + "SupplyChain": 18, + "ExcessivePermissions": 14, + "AtomExhaustion": 11, + "ResourceLeak": 8, + "UnsafeTypeCoercion": 8, + "UncheckedAllocation": 7 + } + }, + "tests": { + "available": true, + "source": "github actions api — active workflow definitions + runs on the default branch", + "definition": "a workflow is a test surface when its name or path matches (test|spec|ci|check|conformance|gate|audit|verify)", + "repos_measured": 408, + "repos_unmeasured": 0, + "with_test_surface": 350, + "coverage_empties": 58, + "coverage_empty_repos": [ + "hyperpolymath/.github", + "hyperpolymath/0patch-lsa-sentinel", + "hyperpolymath/EchoTypes.jl", + "hyperpolymath/action-trust-layers", + "hyperpolymath/aggregate-library", + "hyperpolymath/anvomidav", + "hyperpolymath/awesome-agda", + "hyperpolymath/awesome-haskell", + "hyperpolymath/awesome-idris2", + "hyperpolymath/awesome-ipfs", + "hyperpolymath/awesome-ocaml", + "hyperpolymath/awesome-provable", + "hyperpolymath/bebop-ffi", + "hyperpolymath/befunge93-vault-cracker", + "hyperpolymath/choreographic-types", + "hyperpolymath/claude-gecko-browser-extension", + "hyperpolymath/claude-integrations", + "hyperpolymath/coord-tui", + "hyperpolymath/cyo", + "hyperpolymath/deed-ecosystem", + "hyperpolymath/echo-types", + "hyperpolymath/ensaid-spec", + "hyperpolymath/epistemic-types", + "hyperpolymath/flatracoon", + "hyperpolymath/glyphbase", + "hyperpolymath/gnpl", + "hyperpolymath/gsbot", + "hyperpolymath/homebrew-tap", + "hyperpolymath/hotchocolabot", + "hyperpolymath/hpm-github-api-rsr", + "hyperpolymath/hpm-http-client-rsr", + "hyperpolymath/hpm-json-rsr", + "hyperpolymath/ipv6-site-enforcer", + "hyperpolymath/julia-ecosystem", + "hyperpolymath/me-dialect", + "hyperpolymath/not-so-serious-software", + "hyperpolymath/oikosbot-estate", + "hyperpolymath/pow-the-game", + "hyperpolymath/protocol-squisher", + "hyperpolymath/repo-guardian", + "hyperpolymath/rescript-ecosystem", + "hyperpolymath/resource-record-fluctuator", + "hyperpolymath/scripts", + "hyperpolymath/ssg-collection", + "hyperpolymath/technical-notes", + "hyperpolymath/tentacles-agentic-syllabus", + "hyperpolymath/volumod", + "hyperpolymath/zotero-tools", + "metadatastician/.github", + "metadatastician/authority-watch", + "metadatastician/berrywiki-course-template", + "metadatastician/enterglide-ncc-137-k", + "metadatastician/insolvency-tycoon", + "metadatastician/planer-1", + "metadatastician/selur", + "metadatastician/stealth-glider-ecosystem", + "metadatastician/tardis-rule-disruption", + "metadatastician/x15-rocket-glider" + ], + "failing": 120, + "failing_repos": [ + { + "repo": "hyperpolymath/BowtieRisk.jl", + "workflow": "CI", + "at": "2026-09-25T20:29:14Z" + }, + { + "repo": "hyperpolymath/Causals.jl", + "workflow": "CI", + "at": "2026-09-26T10:34:16Z" + }, + { + "repo": "hyperpolymath/Exnovation.jl", + "workflow": "CI", + "at": "2026-09-26T10:13:47Z" + }, + { + "repo": "hyperpolymath/HardwareResilience.jl", + "workflow": "CI", + "at": "2026-09-25T08:02:29Z" + }, + { + "repo": "hyperpolymath/Hyperpolymath.jl", + "workflow": "CI", + "at": "2026-09-25T13:03:50Z" + }, + { + "repo": "hyperpolymath/JuliaForChildren.jl", + "workflow": "CI", + "at": "2026-09-21T15:43:01Z" + }, + { + "repo": "hyperpolymath/LowLevel.jl", + "workflow": "CI", + "at": "2026-09-21T00:14:25Z" + }, + { + "repo": "hyperpolymath/PolyglotFormalisms.jl", + "workflow": "CI", + "at": "2026-09-21T13:16:24Z" + }, + { + "repo": "hyperpolymath/QuantumCircuit.jl", + "workflow": "CI", + "at": "2026-09-21T01:08:40Z" + }, + { + "repo": "hyperpolymath/SMTLib.jl", + "workflow": "CI", + "at": "2026-09-21T01:13:02Z" + }, + { + "repo": "hyperpolymath/SiliconCore.jl", + "workflow": "CI", + "at": "2026-09-21T01:12:43Z" + }, + { + "repo": "hyperpolymath/SoftwareSovereign.jl", + "workflow": "CI", + "at": "2026-09-21T01:14:50Z" + }, + { + "repo": "hyperpolymath/academic-workflow-suite", + "workflow": "Dogfood Gate", + "at": "2026-09-21T20:59:45Z" + }, + { + "repo": "hyperpolymath/accessibility-everywhere", + "workflow": "AffineScript/Deno CI", + "at": "2026-09-25T08:16:13Z" + }, + { + "repo": "hyperpolymath/ambientops", + "workflow": "Rust CI", + "at": "2026-09-25T20:53:31Z" + }, + { + "repo": "hyperpolymath/anytype", + "workflow": "Dogfood Gate", + "at": "2026-09-25T08:01:50Z" + }, + { + "repo": "hyperpolymath/arghda-core", + "workflow": "Rust CI", + "at": "2026-09-19T09:33:31Z" + }, + { + "repo": "hyperpolymath/asdf-tool-plugins", + "workflow": "Dogfood Gate", + "at": "2026-09-21T20:20:20Z" + }, + { + "repo": "hyperpolymath/awesome-fsharp", + "workflow": ".github/workflows/check-links.yml", + "at": "2026-09-20T00:43:03Z" + }, + { + "repo": "hyperpolymath/awesome-nickel", + "workflow": "Lock Sync Gate", + "at": "2026-09-23T09:41:07Z" + }, + { + "repo": "hyperpolymath/bitfuckit", + "workflow": "CI", + "at": "2026-09-26T10:32:38Z" + }, + { + "repo": "hyperpolymath/blocky-writer", + "workflow": "CI", + "at": "2026-09-26T00:25:56Z" + }, + { + "repo": "hyperpolymath/blue-screen-of-app", + "workflow": "CI/CD", + "at": "2026-09-21T16:23:59Z" + }, + { + "repo": "hyperpolymath/bofig", + "workflow": "Elixir CI", + "at": "2026-09-26T10:34:11Z" + }, + { + "repo": "hyperpolymath/bofj-kitt", + "workflow": "Static Analysis Gate", + "at": "2026-09-26T10:49:14Z" + }, + { + "repo": "hyperpolymath/boinc-boinc", + "workflow": "Central Estate CI/CD Audit", + "at": "2026-09-21T15:59:43Z" + }, + { + "repo": "hyperpolymath/bunsenite", + "workflow": "Rust CI", + "at": "2026-09-25T21:48:39Z" + }, + { + "repo": "hyperpolymath/candy-crash", + "workflow": "CI/CD Pipeline", + "at": "2026-09-21T15:57:49Z" + }, + { + "repo": "hyperpolymath/cloudflare-dns-terraform", + "workflow": "DNS Origin Audit", + "at": "2026-09-26T06:39:35Z" + }, + { + "repo": "hyperpolymath/cloudguard-cli", + "workflow": "Rust CI", + "at": "2026-09-21T20:43:17Z" + }, + { + "repo": "hyperpolymath/cloudguard-server", + "workflow": "Rust CI", + "at": "2026-09-21T20:52:31Z" + }, + { + "repo": "hyperpolymath/conflow", + "workflow": "Rust CI", + "at": "2026-09-21T20:44:56Z" + }, + { + "repo": "hyperpolymath/cookie-rebound", + "workflow": "Static Analysis Gate", + "at": "2026-09-21T20:21:12Z" + }, + { + "repo": "hyperpolymath/coq-jr", + "workflow": "Security Checks", + "at": "2026-09-21T16:44:05Z" + }, + { + "repo": "hyperpolymath/ddraig-ssg", + "workflow": "Ddraig CI", + "at": "2026-09-26T10:50:13Z" + }, + { + "repo": "hyperpolymath/deed-core", + "workflow": "ReScript Tests", + "at": "2026-09-21T08:23:49Z" + }, + { + "repo": "hyperpolymath/dicti0nary-attack", + "workflow": "Rust CI", + "at": "2026-09-21T01:38:53Z" + }, + { + "repo": "hyperpolymath/docmatrix", + "workflow": "Rust CI", + "at": "2026-09-21T20:14:43Z" + }, + { + "repo": "hyperpolymath/dotmatrix-fileprinter", + "workflow": "Rust CI", + "at": "2026-09-23T09:42:20Z" + }, + { + "repo": "hyperpolymath/double-track-browser", + "workflow": "CI", + "at": "2026-09-22T06:54:25Z" + }, + { + "repo": "hyperpolymath/echidnabot", + "workflow": "Rust CI", + "at": "2026-09-25T12:10:27Z" + }, + { + "repo": "hyperpolymath/excel-economic-numbers-tool", + "workflow": "Integration Tests", + "at": "2026-09-26T00:02:19Z" + }, + { + "repo": "hyperpolymath/filesoup", + "workflow": "Rust CI", + "at": "2026-09-21T20:47:34Z" + }, + { + "repo": "hyperpolymath/flat-mate", + "workflow": "CI", + "at": "2026-09-21T19:14:05Z" + }, + { + "repo": "hyperpolymath/formatrix-docs", + "workflow": "Rust CI", + "at": "2026-09-21T20:45:34Z" + }, + { + "repo": "hyperpolymath/fraying-model-computational-testbed", + "workflow": "Central Estate CI/CD Audit", + "at": "2026-09-21T19:38:28Z" + }, + { + "repo": "hyperpolymath/git-reticulator", + "workflow": "Rust CI", + "at": "2026-09-21T20:17:58Z" + }, + { + "repo": "hyperpolymath/git-scripts", + "workflow": ".github/workflows/main-estate-audit.yml", + "at": "2026-09-24T06:32:28Z" + }, + { + "repo": "hyperpolymath/grim-repo", + "workflow": "Dogfood Gate", + "at": "2026-09-26T09:48:02Z" + }, + { + "repo": "hyperpolymath/gv-clade-index", + "workflow": "Static Analysis Gate", + "at": "2026-09-21T21:02:49Z" + }, + { + "repo": "hyperpolymath/hesiod-dns-map", + "workflow": "Rust CI", + "at": "2026-09-21T20:54:00Z" + }, + { + "repo": "hyperpolymath/ideas-to-alphas", + "workflow": "Static Analysis Gate", + "at": "2026-09-21T20:25:07Z" + }, + { + "repo": "hyperpolymath/infrastructure-automation", + "workflow": "RSR Anti-Pattern Check", + "at": "2026-09-19T10:50:57Z" + }, + { + "repo": "hyperpolymath/intsoc-transactor", + "workflow": "Rust CI", + "at": "2026-09-21T20:48:38Z" + }, + { + "repo": "hyperpolymath/januskey", + "workflow": "Rust CI", + "at": "2026-09-24T06:32:59Z" + }, + { + "repo": "hyperpolymath/julia-professional-registry", + "workflow": "Lock Sync Gate", + "at": "2026-09-23T09:37:37Z" + }, + { + "repo": "hyperpolymath/julianiser", + "workflow": "Rust CI", + "at": "2026-09-25T20:27:52Z" + }, + { + "repo": "hyperpolymath/kategoria", + "workflow": "Proof Gate", + "at": "2026-09-21T18:00:43Z" + }, + { + "repo": "hyperpolymath/kea", + "workflow": "Dogfood Gate", + "at": "2026-09-21T20:26:02Z" + }, + { + "repo": "hyperpolymath/lithoglyph", + "workflow": "Static Analysis Gate", + "at": "2026-09-22T23:38:12Z" + } + ], + "never_concluded": 17, + "last_run": "2026-09-26T10:52:36Z" + }, + "benches": { + "available": false, + "reason": "no bench-result producer anywhere in the estate — no store, no workflow artefact contract", + "expected": { + "per_repo_file": "benches.json", + "fields": [ + "repo", + "bench", + "value_ns", + "limit_ns", + "measured_at" + ] + } + } + }, + "paging": { + "open_issue_count": { + "value": 887, + "warn": 250, + "critical": 500, + "level": "critical" + }, + "unmerged_pr_count": { + "value": 33, + "warn": 10, + "critical": 50, + "level": "warn" + }, + "unmerged_pr_age_days": { + "value": 7, + "warn": 7, + "critical": 30, + "level": "warn" + }, + "failing_tests": { + "value": 120, + "warn": 1, + "critical": 25, + "level": "critical" + }, + "test_coverage_empties": { + "value": 58, + "warn": 11, + "critical": 47, + "level": "critical" + }, + "bench_coverage_empties": { + "value": null, + "warn": 11, + "critical": 47, + "level": "unavailable" + }, + "bench_over_limit_ratio": { + "value": null, + "warn": 1.5, + "critical": 3, + "level": "unavailable" + } + } +} From 831af4cc2a73a30417db782f1c741809d97da85a Mon Sep 17 00:00:00 2001 From: hypatia Date: Sat, 26 Sep 2026 14:57:06 +0000 Subject: [PATCH 5/6] feat(sweeps): hold the dependabot generator, not just the symptom MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repairing a pin does not stop the pin coming back, and the measurement says so: 133 poisoned files across 104 repositories; 337 repositories estate-wide carry the dependabot shape that lets the bump through; of the 104 that already carry the poison, 102 are exposed, 0 hold the action, and 31 hold it in the form standards#1037 established is not honoured inside `groups:`. The evidence that this is not theoretical: nexia-list merged the poisoned bump in #107 on 2026-09-23 and still carries the poisoned commit today, annotated with the number of the pull request that was supposed to have rolled it back. estate-dependabot-hold.sh installs the hold that survives grouping — `exclude-patterns` on the group, scoped to the action rather than holding the whole group. Two added lines per file, nothing reordered, nothing reflowed. Idempotent, and `--verify-only` exits non-zero while anything is still exposed, so the same script is both the cure and the check that the cure is holding. Verified against the estate: 102 of 102 patched files parse as YAML and carry the exclusion; 2 repositories have no wildcard group to hang it on and are reported for a human rather than edited blind. The sweep now proves a patch parses before writing it — the first attempt silently emitted a column-0 entry in one file out of 102, which is exactly the failure mode a sweep must not have. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/tests.yml | 2 +- docs/operations/estate-automerge.adoc | 86 +++++ scripts/sweeps/estate-dependabot-hold.sh | 381 +++++++++++++++++++++++ 3 files changed, 468 insertions(+), 1 deletion(-) create mode 100755 scripts/sweeps/estate-dependabot-hold.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6e62f5b0..57b3251c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,7 +67,7 @@ jobs: - name: Sweep structure gate run: | set -euo pipefail - for s in estate-pin-integrity estate-pr-automerge estate-stats estate-absence-intake; do + for s in estate-pin-integrity estate-pr-automerge estate-stats estate-absence-intake estate-dependabot-hold; do bash -n "scripts/sweeps/${s}.sh" echo "ok: ${s}.sh" done diff --git a/docs/operations/estate-automerge.adoc b/docs/operations/estate-automerge.adoc index 91d2be5f..db17e4d0 100644 --- a/docs/operations/estate-automerge.adoc +++ b/docs/operations/estate-automerge.adoc @@ -284,6 +284,92 @@ gate on checks it can actually run itself — the `dynamic`-event PR validation runs do execute (`PR #72`, `PR #107`) — and must treat GitHub's own pull request checks on its own pull requests as **unavailable, never as passing**. +== Why this recurs, and what stops it + +Repairing a pin is not the same as stopping the pin coming back. The census on +2026-09-26, over the repositories that carry the poisoned commit: + +[cols="1,4", options="header"] +|=== +| Count | State + +| 133 files, 104 repositories +| carry `github/codeql-action@1c5b6756…` — the commit GitHub refuses at + workflow start-up + +| 103 of 104 +| use dependabot `groups:` on the `github-actions` ecosystem + +| 73 of 104 +| have **no** `ignore` rule for the action at all + +| 31 of 104 +| have `ignore: - dependency-name: "github/codeql-action"` — the rule + standards#1037 established is **not honoured inside `groups:`** + +| 0 of 104 +| have the form that is honoured +|=== + +So the generator is still on. Closing a poisoned pull request without holding +the action does not end it: dependabot re-raises the bump on its next run, and +the estate gets to sort the same issue again next week. That is not a +residual risk; it is the observed behaviour — `nexia-list` merged the poisoned +bump in #107 on 2026-09-23, and its `codeql.yml` still carries the poisoned +commit today, annotated with the rollback pull request's number. + +`scripts/sweeps/estate-dependabot-hold.sh` installs the hold that survives +grouping — `exclude-patterns` on the group, scoped to the action rather than +holding the whole group: + +[source,yaml] +---- + groups: + actions: + patterns: + - "*" + exclude-patterns: + - "github/codeql-action*" +---- + +* Two added lines per file, nothing reordered, nothing reformatted — a diff + that can be reviewed rather than trusted. +* Idempotent: a second `--rewrite` over a repository it already fixed reports + `held`, not a second edit. +* `--verify-only` exits non-zero while any repository is still exposed, so the + same script is both the cure and the check that the cure is holding. That is + the difference between a fix and a chore. +* It refuses to guess: a repository with no `github-actions` wildcard group to + attach the exclusion to is reported as `no_group_wildcard` for a human, + never edited blind. + +The template is the second generator. `rsr-template-repo` carries the correct +pin, so new repositories are not born poisoned — but its `dependabot.yml` +carries the *bypassed* form of the hold, so they are born exposed. Fixing the +existing repositories without fixing the template just sets the clock running +again on the next repository created. + +The third generator was historical: repairs applied as relabels. That one is +now visible (`PI002`) rather than a matter of trust. + +=== What "once and for all" requires + +. Hold the generator — `estate-dependabot-hold.sh --rewrite`, one command, + ~104 two-line patches. +. Fix the template's `dependabot.yml`, or every new repository inherits the + exposure. +. Repair the standing poison — `estate-pin-integrity.sh --rewrite`. +. Enforce at the boundary: run `estate-dependabot-hold.sh --verify-only` (and + the pin rules) as a required check, so the next bad pin cannot merge even if + it is proposed. This is the piece that needs repository-admin rights; no + sweep can grant itself that. +. Keep the sweep scheduled and the statistics artifact committed, so the + estate's state is a recorded fact rather than anyone's recollection. + +Steps 1–3 are mechanical and reproducible. Step 4 is the one that turns +"fixed" into "cannot drift". Without it, the estate is one dependabot run and +one distracted afternoon away from the same Friday. + == Not yet implemented Three things are specified in the policy but not yet built, listed here so diff --git a/scripts/sweeps/estate-dependabot-hold.sh b/scripts/sweeps/estate-dependabot-hold.sh new file mode 100755 index 00000000..c54ac373 --- /dev/null +++ b/scripts/sweeps/estate-dependabot-hold.sh @@ -0,0 +1,381 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# estate-dependabot-hold.sh — stop the generator, not just the symptom. +# +# WHY THIS IS NOT THE PIN SWEEP +# +# estate-pin-integrity.sh repairs the *symptom*: it removes the poisoned pin +# from a file. But the pin came back twice because something keeps proposing it. +# On 2026-09-26 the census was: +# +# 133 poisoned files across 104 repositories +# 103 of those 104 use dependabot `groups:` +# 73 have no `ignore` rule for the action at all +# 31 have `ignore: - dependency-name: "github/codeql-action"` +# 0 have the cure +# +# and the 31 are the trap, not the protection: standards#1037 established that +# an update-level `ignore` is NOT honoured inside `groups:` — the poisoned bump +# arrives anyway, SHA-swapped, inline comment and all. A closed pull request +# without a hold is a pull request you will see again next week. +# +# Dependabot's group config has `exclude-patterns` for exactly this. It is the +# only form of the hold that survives grouping, and it is scoped to the action +# rather than holding the whole group (a blanket `*` hold is a maintenance +# hostage: it stops every unrelated bump too). +# +# WHAT IT EDITS +# +# One insertion, under the wildcard pattern of a `github-actions` group: +# +# patterns: +# - "*" +# + exclude-patterns: +# + - "github/codeql-action*" +# +# Nothing else is touched. The existing `ignore` rule is left in place: it is +# harmless, and it is what protects any update that is not part of a group. +# If the anchor cannot be found the repository is reported as a finding, never +# guessed at — a repair that cannot be proved is not a repair. +# +# IDEMPOTENT BY CONSTRUCTION. Run it twice: the second run reports `ok` for +# every repository it rewrote. That is what makes this a one-time fix rather +# than a chore: --verify-only exits 1 when any repository is still exposed, so +# the same script is both the cure and the check that the cure is holding. +# +# EXIT CODES +# 0 nothing to do / all verified +# 1 error (bad arguments, missing tooling) +# 2 work to do (dry run with exposed repositories), --rewrite or --verify-only +# 3 no repositories scanned +# +# usage: estate-dependabot-hold.sh [--org ORG]... [--policy FILE] [--out DIR] +# [--repo OWNER/NAME] [--only-file FILE] +# [--rewrite] [--verify-only] [--max-repos N] +# +# --only-file FILE restrict to repositories listed one per line. Use the +# pin sweep's census ("the repositories that actually +# carry the poison") when you do not want a defensive +# hold in all 300+ repositories that merely have the +# vulnerable dependabot shape. +set -euo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +POLICY="$REPO_ROOT/.machine_readable/merge-orchestration/pr-automerge-policy.json" +OUT_DIR="${PWD}/.dependabot-hold" +ORGS=() +ONE_REPO="" +ONLY_FILE="" +REWRITE=0 +VERIFY_ONLY=0 +MAX_REPOS=0 + +while [ $# -gt 0 ]; do + case "$1" in + --org) ORGS+=("$2"); shift 2 ;; + --policy) POLICY="$2"; shift 2 ;; + --out) OUT_DIR="$2"; shift 2 ;; + --repo) ONE_REPO="$2"; shift 2 ;; + --only-file) ONLY_FILE="$2"; shift 2 ;; + --rewrite) REWRITE=1; shift ;; + --verify-only) VERIFY_ONLY=1; shift ;; + --max-repos) MAX_REPOS="$2"; shift 2 ;; + -h|--help) sed -n '2,52p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "unknown argument: $1" >&2; exit 1 ;; + esac +done + +[ ${#ORGS[@]} -eq 0 ] && ORGS=("hyperpolymath" "metadatastician") +[ -f "$POLICY" ] || { echo "policy not found: $POLICY" >&2; exit 1; } +command -v gh >/dev/null || { echo "gh is required" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 1; } +command -v python3 >/dev/null || { echo "python3 is required (YAML edit)" >&2; exit 1; } + +mkdir -p "$OUT_DIR" +findings="$OUT_DIR/hold-findings.jsonl" +plan="$OUT_DIR/hold-plan.jsonl" +: > "$findings" +: > "$plan" + +log() { printf '%s\n' "$*" >&2; } +cleanup() { rm -f "$OUT_DIR"/.db.*.tmp 2>/dev/null || true; } + +# The action and the glob that excludes it, from the policy — one source of +# truth, so a second denylisted action is a policy edit and not a code change. +mapfile -t HELD_ACTIONS < <(jq -r '.pin_denylist[] | select((.blocked_shas | length) > 0) | .action' "$POLICY") +[ ${#HELD_ACTIONS[@]} -gt 0 ] || { echo "policy denylist is empty" >&2; exit 1; } + +GLOBS=() +for a in "${HELD_ACTIONS[@]}"; do GLOBS+=("${a}*"); done +GLOB_JSON=$(printf '%s\n' "${GLOBS[@]}" | jq -Rsc 'split("\n") | map(select(. != ""))') + +log "policy $(jq -r '.version' "$POLICY") · holding ${#HELD_ACTIONS[@]} action(s) · orgs ${ORGS[*]}" + +list_repos() { + local org="$1" + gh api "orgs/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | "\(.name)"' 2>/dev/null \ + || gh api "users/${org}/repos?per_page=100" --paginate \ + --jq '.[] | select(.archived == false and .fork == false and .disabled == false) | "\(.name)"' 2>/dev/null +} + +# ─── The edit ──────────────────────────────────────────────────────────── +# Deliberately a line-oriented insertion rather than a YAML round-trip: a +# re-serialised dependabot.yml would reorder keys and reformat comments in +# every repository, which makes the diff unreviewable and the sweep +# unusable. This touches exactly two lines. +edit_dependabot() { + local file="$1" globs_json="$2" + python3 - "$file" "$globs_json" <<'PY' +import json, sys + +path, globs_json = sys.argv[1], sys.argv[2] +globs = json.loads(globs_json) + +with open(path) as fh: + raw = fh.read() + +original_had_newline = raw.endswith("\n") +lines = raw.split("\n") +if lines and lines[-1] == "": + # split() leaves a trailing empty element; the join below restores it. + lines = lines[:-1] + + +def indent_of(line): + return len(line) - len(line.lstrip(" ")) + + +# Find every `patterns:` list inside a groups: block and take the ones whose +# list contains a bare "*", i.e. the group that swallows everything. +anchors = [] +for i, line in enumerate(lines): + if line.strip() != "patterns:": + continue + base = indent_of(line) + j = i + 1 + has_wildcard = False + last_content = i + while j < len(lines) and (lines[j].strip() == "" or indent_of(lines[j]) > base): + if lines[j].strip() != "": + last_content = j + s = lines[j].strip() + if s in ('- "*"', "- '*'", "- *"): + has_wildcard = True + j += 1 + if not has_wildcard: + continue + # Only groups inside a github-actions update: walk back to the nearest + # `- package-ecosystem:` line. + ecosystem = None + k = i + while k >= 0: + if lines[k].strip().startswith("- package-ecosystem:"): + ecosystem = lines[k].split(":", 1)[1].strip().strip('"\'') + break + k -= 1 + if ecosystem == "github-actions": + # Attach to the last real entry, never to a trailing blank line. + anchors.append((i, last_content, base)) + +if not anchors: + print("NO_ANCHOR") + sys.exit(3) + +# Insert after the last wildcard entry of each anchor, deepest first so the +# earlier indices stay valid. +inserted = 0 +for _, last_idx, base in reversed(anchors): + # Align the new entry with the wildcard entry above it, not with some + # indentation of our own: the diff should look like the file's own style. + item_indent = indent_of(lines[last_idx]) + block = [f"{' ' * max(base, 0)}exclude-patterns:"] + block += [f"{' ' * item_indent}- \"{g}\"" for g in globs] + if any("exclude-patterns:" in ln for ln in lines[last_idx + 1:last_idx + 4]): + continue + lines[last_idx + 1:last_idx + 1] = block + inserted += 1 + +if inserted == 0: + print("ALREADY_HELD") + sys.exit(4) + +text = "\n".join(lines) +if original_had_newline and not text.endswith("\n"): + text += "\n" + +# Prove the patch before writing it. This is the check that caught an anchor +# attached to a trailing blank line silently emitting an entry at column 0 in +# a 102-file run — one broken file among a hundred is exactly the kind of +# thing a sweep must not do. +try: + import yaml # noqa: F401 +except ImportError: + yaml = None + +if yaml is not None: + try: + yaml.safe_load(text) + except Exception as exc: # pragma: no cover - depends on the file + print("EDIT_UNPARSEABLE " + str(exc).split("\n")[0]) + sys.exit(5) + +with open(path, "w") as fh: + fh.write(text) + +print(f"HELD {inserted}") +PY +} + +# ─── Scan ──────────────────────────────────────────────────────────────── +exposed=0 +held=0 +acked=0 +scanned=0 + +trap cleanup EXIT + +# The list is built first and the loop reads it as a redirect, not a pipeline: +# a piped loop runs in a subshell, and a subshell is how a sweep reports +# "nothing to do" while holding a file full of findings. +if [ -n "$ONLY_FILE" ]; then + [ -s "$ONLY_FILE" ] || { echo "only-file is empty or missing: $ONLY_FILE" >&2; exit 1; } + grep -v '^[[:space:]]*$' "$ONLY_FILE" | sort -u > "$OUT_DIR/repos.txt" +elif [ -n "$ONE_REPO" ]; then + printf '%s\n' "$ONE_REPO" > "$OUT_DIR/repos.txt" +else + : > "$OUT_DIR/repos.txt" + for org in "${ORGS[@]}"; do + list_repos "$org" | sed "s|^|${org}/|" >> "$OUT_DIR/repos.txt" + done +fi +log "repositories to check: $(wc -l < "$OUT_DIR/repos.txt")" + +while read -r full; do + [ -n "$full" ] || continue + scanned=$((scanned + 1)) + [ $((scanned % 25)) -eq 0 ] && log " … ${scanned}" + + # Fetch to a file, not into a variable: `$(...)` strips trailing newlines, + # and a rewritten file that silently loses its final newline is a diff + # nobody will trust. + db="" + db_tmp="$OUT_DIR/.db.$$.tmp" + : > "$db_tmp" + for p in .github/dependabot.yml .github/dependabot.yaml dependabot.yml; do + if gh api "repos/${full}/contents/${p}" --jq '.content' 2>/dev/null | base64 -d > "$db_tmp" 2>/dev/null && [ -s "$db_tmp" ]; then + db="$p"; break + fi + done + content=$(cat "$db_tmp") + + if [ -z "$db" ]; then + jq -cn --arg repo "$full" '{repo:$repo, status:"no_dependabot", detail:"no dependabot.yml anywhere — nothing to hold"}' >> "$plan" + continue + fi + + cures=$(printf '%s' "$content" | grep -c 'exclude-patterns' || true) + gulps=$(printf '%s' "$content" | grep -cE '^\s+- "\*"|^\s+- .\*.$' || true) + + if [ "$cures" -gt 0 ]; then + held=$((held + 1)) + jq -cn --arg repo "$full" --arg path "$db" \ + '{repo:$repo, path:$path, status:"held", detail:"exclude-patterns present"}' >> "$plan" + continue + fi + + if [ "$gulps" -eq 0 ]; then + # No wildcard group: a plain `ignore` is honoured here, so this is a + # judgement call, not a mechanical edit. + acked=$((acked + 1)) + jq -cn --arg repo "$full" --arg path "$db" \ + '{repo:$repo, path:$path, status:"no_group_wildcard", detail:"no grouped wildcard; the update-level ignore is honoured — verify by hand"}' >> "$plan" + continue + fi + + exposed=$((exposed + 1)) + if [ "$REWRITE" -eq 1 ]; then + mkdir -p "$OUT_DIR/rewritten/${full}/$(dirname "$db")" + cp "$db_tmp" "$OUT_DIR/rewritten/${full}/${db}" + result=$(edit_dependabot "$OUT_DIR/rewritten/${full}/${db}" "$GLOB_JSON" 2>&1 || true) + case "$result" in + HELD*) + jq -cn --arg repo "$full" --arg path "$db" --arg r "$result" \ + '{repo:$repo, path:$path, status:"rewritten", detail:$r}' >> "$plan" ;; + ALREADY_HELD) + jq -cn --arg repo "$full" --arg path "$db" \ + '{repo:$repo, path:$path, status:"held", detail:"already carried the exclusion"}' >> "$plan" ;; + EDIT_UNPARSEABLE*) + jq -cn --arg repo "$full" --arg path "$db" --arg r "$result" \ + '{repo:$repo, path:$path, status:"flag", detail:("edit would not parse, left untouched: " + $r)}' >> "$plan" + # Put the original back: an unparseable dependabot.yml is worse than + # an exposed one. + gh api "repos/${full}/contents/${db}" --jq '.content' 2>/dev/null | base64 -d > "$OUT_DIR/rewritten/${full}/${db}" 2>/dev/null || true ;; + NO_ANCHOR) + jq -cn --arg repo "$full" --arg path "$db" \ + '{repo:$repo, path:$path, status:"flag", detail:"no github-actions wildcard group to hang the exclusion on — needs a human"}' >> "$plan" + rm -rf "$OUT_DIR/rewritten/${full}" ;; + *) + jq -cn --arg repo "$full" --arg path "$db" --arg r "$result" \ + '{repo:$repo, path:$path, status:"flag", detail:("edit failed: " + $r)}' >> "$plan" + rm -rf "$OUT_DIR/rewritten/${full}" ;; + esac + else + jq -cn --arg repo "$full" --arg path "$db" \ + '{repo:$repo, path:$path, status:"exposed", detail:"wildcard group with no exclusion — dependabot will re-raise the held action here"}' >> "$plan" + fi +done < "$OUT_DIR/repos.txt" + +# The loop above writes the plan; the summary reads it back so the two can +# never disagree. +summary() { + jq -s ' + { + repositories_scanned: length, + by_status: (group_by(.status) | map({key: .[0].status, value: length}) | from_entries), + held: [.[] | select(.status == "held") | .repo], + exposed: [.[] | select(.status == "exposed" or .status == "rewritten") | .repo], + flags: [.[] | select(.status == "flag" or .status == "no_group_wildcard") | {repo, detail}] + }' "$plan" +} + +log "" +log "─── dependabot hold ───────────────────────────────────────────────" +summary > "$OUT_DIR/hold-summary.json" +jq -r '" scanned : \(.repositories_scanned)\n already held : \(.by_status.held // 0)\n exposed : \((.exposed | length))\n flagged for hand : \((.flags | length))"' "$OUT_DIR/hold-summary.json" >&2 +log " plan : ${plan}" +log " summary : ${OUT_DIR}/hold-summary.json" + +if [ "$VERIFY_ONLY" -eq 1 ]; then + n_exposed=$(jq -r '(.exposed | length)' "$OUT_DIR/hold-summary.json") + if [ "$n_exposed" -gt 0 ]; then + log "" + log " VERIFY FAILED: ${n_exposed} repositories can still receive the held action." + exit 2 + fi + log "" + log " VERIFY OK: every scanned repository holds the pinned action." + exit 0 +fi + +if [ "$REWRITE" -eq 1 ]; then + log "" + log " rewritten trees : ${OUT_DIR}/rewritten/ (one dependabot.yml per repository)" + log " next : review a diff, then branch + PR per repository" + exit 0 +fi + +if [ "$exposed" -eq 0 ]; then + log "" + log " nothing to do — every scanned repository already holds the action." + exit 0 +fi + +log "" +log " dry run — nothing written. Pass --rewrite to produce the corrected" +log " dependabot.yml files, or --verify-only to use this as a check." +exit 2 From d48255315fc5bac94a75ec1e0676cf5a27ae7330 Mon Sep 17 00:00:00 2001 From: "coderabbitai[bot]" <136622811+coderabbitai[bot]@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:27:50 +0000 Subject: [PATCH 6/6] docs(sweeps): document dependabot hold helper behavior and error handling --- scripts/sweeps/estate-dependabot-hold.sh | 28 ++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/scripts/sweeps/estate-dependabot-hold.sh b/scripts/sweeps/estate-dependabot-hold.sh index c54ac373..f5b7900c 100755 --- a/scripts/sweeps/estate-dependabot-hold.sh +++ b/scripts/sweeps/estate-dependabot-hold.sh @@ -101,7 +101,9 @@ plan="$OUT_DIR/hold-plan.jsonl" : > "$findings" : > "$plan" +# Write a message to standard error, followed by a newline. log() { printf '%s\n' "$*" >&2; } +# Remove .db.*.tmp files from OUT_DIR on exit, ignoring removal failures. cleanup() { rm -f "$OUT_DIR"/.db.*.tmp 2>/dev/null || true; } # The action and the glob that excludes it, from the policy — one source of @@ -115,6 +117,9 @@ GLOB_JSON=$(printf '%s\n' "${GLOBS[@]}" | jq -Rsc 'split("\n") | map(select(. != log "policy $(jq -r '.version' "$POLICY") · holding ${#HELD_ACTIONS[@]} action(s) · orgs ${ORGS[*]}" +# Print repository names, one per line, for the organisation or user in $1, +# excluding archived, forked and disabled repositories. If the organisation +# request fails, try the user endpoint; return its failure status if it fails. list_repos() { local org="$1" gh api "orgs/${org}/repos?per_page=100" --paginate \ @@ -124,10 +129,20 @@ list_repos() { } # ─── The edit ──────────────────────────────────────────────────────────── -# Deliberately a line-oriented insertion rather than a YAML round-trip: a -# re-serialised dependabot.yml would reorder keys and reformat comments in -# every repository, which makes the diff unreviewable and the sweep -# unusable. This touches exactly two lines. +# Insert exclude-patterns into the local file in $1 using the JSON array of +# action globs in $2. Match patterns lists containing a bare wildcard whose +# nearest preceding package-ecosystem entry is github-actions. Skip a list +# when exclude-patterns occurs in the next three lines after its last content. +# Insert lines without re-serialising the YAML; each insertion adds one key +# line and one entry per glob. +# +# On success, overwrite the file, print HELD followed by the insertion count +# and return 0. Leave the file untouched and print NO_ANCHOR (status 3) if no +# list matches, or ALREADY_HELD (4) if all matching lists are skipped. When +# PyYAML is available, reject invalid output before writing: print +# EDIT_UNPARSEABLE with the first error line and return 5. Without PyYAML, +# write without YAML validation. JSON decoding and file I/O errors propagate +# as Python diagnostics and a non-zero status; a failed write may be partial. edit_dependabot() { local file="$1" globs_json="$2" python3 - "$file" "$globs_json" <<'PY' @@ -147,6 +162,7 @@ if lines and lines[-1] == "": def indent_of(line): + """Return the number of leading spaces, stopping at tabs or other characters.""" return len(line) - len(line.lstrip(" ")) @@ -332,6 +348,10 @@ done < "$OUT_DIR/repos.txt" # The loop above writes the plan; the summary reads it back so the two can # never disagree. +# Print a JSON summary of the records in $plan, with counts by status, held +# repository names, exposed names (including rewritten repositories), and +# details for flag and no_group_wildcard records. Return jq's status, including +# failures to read or parse the plan. summary() { jq -s ' {