diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 70ce59b..5700dec 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,6 +1,6 @@ # `.github/workflows/` — CI workflows -> Eight GitHub Actions workflows that wire the agentile rules and +> Nine GitHub Actions workflows that wire the agentile rules and > ratchets into PR-time enforcement. Each workflow invokes a script > from `scripts/ci/` or `scripts/semgrep/`; the workflow itself only > handles GitHub-side glue (when to run, what to check out, how to @@ -18,11 +18,12 @@ sends PR text to a third-party LLM endpoint. | `lint-frontmatter.yml` | PR / push to .md | `check_frontmatter.py` on changed `.md` files | 12 | `contents: read` | — | | `lint-workflows.yml` | PR / push to `.github/workflows/**` | `actionlint` on workflow YAML | (supply chain hygiene) | `contents: read` | — | | `ratchet-check.yml` | every PR / push to main | All four ratchets | 3, 10, 12, tripwire discipline | `contents: read` | — | -| `tripwires.yml` | every PR / push to main | `check_no_unwraps`, `check_no_mocks`, Semgrep, frontmatter-on-added | 2, 5, 11, 12 | `contents: read` | — | +| `tripwires.yml` | every PR / push to main | `check_no_unwraps`, `check_no_mocks`, Semgrep, frontmatter-on-added, the `tests/` regression suite | 2, 5, 11, 12 | `contents: read` | — | | `audit-immutability.yml` | PR / push to `.agentile/audits/**` | No modification to existing audit files | 6 | `contents: read` | — | | `claim-grade.yml` | PR / `workflow_dispatch` | AI claim-compression grader (shadow mode) | (honesty gate) | `contents: read`, **`pull-requests: write`** | **`ANTHROPIC_API_KEY` / `OPENAI_API_KEY`** (sends PR text off-platform) | | `data-source-check.yml` | PR / `workflow_dispatch` | Data-source provenance check (shadow mode) | (Rule 11 support) | `contents: read`, **`pull-requests: write`** | — | | `benchmark-nightly.yml` | schedule / `workflow_dispatch` | Nightly benchmark harness | (perf regression) | `contents: read` | — | +| `merge-gate.yml` | every PR / push to main / merge queue | Producers for the two required checks: `secret-scan` (pinned gitleaks over the change) and `ci` (fails if any other check on the commit failed) | (merge gate) | `contents: read` (+ `checks: read` on the `ci` job) | — | ## Non-GitHub CI @@ -72,8 +73,8 @@ cancel each other. ## Permissions Every workflow declares an explicit top-level `permissions:` block — -none inherit the default token scope. Six workflows are -`contents: read` only. **Two — `claim-grade.yml` and +none inherit the default token scope. Seven workflows are +`contents: read` only (`merge-gate.yml`'s `ci` job adds `checks: read`). **Two — `claim-grade.yml` and `data-source-check.yml` — additionally declare `pull-requests: write`** so they can post a comment on the PR. **`claim-grade.yml` also consumes `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` and sends PR title/body/commit diff --git a/.github/workflows/tripwires.yml b/.github/workflows/tripwires.yml index a83c3d1..c632065 100644 --- a/.github/workflows/tripwires.yml +++ b/.github/workflows/tripwires.yml @@ -46,6 +46,28 @@ jobs: - name: Run check_no_mocks.py run: ./scripts/ci/check_no_mocks.py + self-tests: + name: Skeleton regression tests (tests/) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: '3.11' + + # The skeleton's own regression suite (fail-closed grader, ratchet + # vacuity, bootstrap staging, hook portability, workflow hygiene). + # Skipped cleanly in a project that removed tests/. + - name: Run tests/ + run: | + if [ -d tests ]; then + python -m unittest discover -s tests -p 'test_*.py' -v + else + echo "no tests/ directory; nothing to run" + fi + semgrep: name: Semgrep tripwires (AST-grade) runs-on: ubuntu-latest diff --git a/bootstrap.sh b/bootstrap.sh index d59c3b8..02de81e 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -264,8 +264,10 @@ else fi # Also seed .claude/settings.json from template (gitignored target). +seeded_settings=0 if [[ ! -f "$PROJECT_ROOT/.claude/settings.json" ]] && [[ -f "$PROJECT_ROOT/.claude/settings.json.template" ]]; then cp "$PROJECT_ROOT/.claude/settings.json.template" "$PROJECT_ROOT/.claude/settings.json" + seeded_settings=1 echo " Seeded .claude/settings.json from template (gitignored)." fi @@ -274,6 +276,7 @@ fi # --------------------------------------------------------------------- placeholder_dir="$AGENTILE/sprints/active/YYYY-XX-XX-sprint-0-bootstrap" +sprint_rel="" if [[ -d "$placeholder_dir" ]]; then new_dir="$AGENTILE/sprints/active/${DAY_ZERO_DATE}-sprint-0-bootstrap" if [[ ! -e "$new_dir" ]]; then @@ -289,6 +292,7 @@ if [[ -d "$placeholder_dir" ]]; then sed -i.bak "s/| \*\*Start Date\*\* | YYYY-MM-DD |/| **Start Date** | ${DAY_ZERO_DATE} |/" "$sprint_md" rm -f "$sprint_md.bak" fi + sprint_rel=".agentile/sprints/active/${DAY_ZERO_DATE}-sprint-0-bootstrap" fi else echo "[5/9] No Sprint 0 placeholder to rename." @@ -387,7 +391,28 @@ fi # --------------------------------------------------------------------- echo "[9/9] Staging bootstrap changes ..." -git -C "$PROJECT_ROOT" add -A 2>/dev/null || true +# Stage only the paths this script creates or updates. Never stage the +# adopter's whole tree: it may hold credentials or other local files. +stage_bootstrap_path() { + local rel_path="$1" + if [[ -e "$PROJECT_ROOT/$rel_path" || -L "$PROJECT_ROOT/$rel_path" ]]; then + git -C "$PROJECT_ROOT" add -A -- "$rel_path" || echo " WARNING: could not stage $rel_path" >&2 + fi +} +stage_bootstrap_path ".agentile/CONFIG.md" +stage_bootstrap_path ".agentile/PRODUCT_SPEC.md" +stage_bootstrap_path ".agentile/coverage/BASELINE.md" +stage_bootstrap_path ".agentile/coverage/baseline.json" +stage_bootstrap_path ".agentile/INDEX" +stage_bootstrap_path "CLAUDE.md" +if [[ "$seeded_settings" -eq 1 ]]; then + stage_bootstrap_path ".claude/settings.json" +fi +if [[ -n "$sprint_rel" ]]; then + stage_bootstrap_path "$sprint_rel" + # Record the placeholder's removal too (covers the plain-mv fallback). + git -C "$PROJECT_ROOT" add -A -- ".agentile/sprints/active/YYYY-XX-XX-sprint-0-bootstrap" 2>/dev/null || true +fi if [[ -z "$(git -C "$PROJECT_ROOT" diff --cached --name-only 2>/dev/null)" ]]; then echo " Nothing to commit." else diff --git a/tests/test_bootstrap_staging.py b/tests/test_bootstrap_staging.py new file mode 100644 index 0000000..531cd2c --- /dev/null +++ b/tests/test_bootstrap_staging.py @@ -0,0 +1,99 @@ +"""bootstrap.sh must commit only the files it generates. + +Runs the real ``bootstrap.sh --non-interactive`` inside a throwaway copy of +the skeleton (its own git repo, HOME and git config isolated) with unrelated +files planted in the adopter's tree, then inspects what was committed and +what is staged. Nothing outside the temp dir is touched. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] + + +def _git(cwd: Path, *args: str, env: dict) -> str: + return subprocess.run( + ["git", *args], cwd=cwd, env=env, check=True, capture_output=True, text=True + ).stdout + + +class BootstrapStagesOnlyGeneratedFiles(unittest.TestCase): + PLANTED = ("local-notes.txt", "private/creds.json") + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix="ag-boot-")) + self.proj = self.tmp / "proj" + home = self.tmp / "home" + home.mkdir() + self.env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(home), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "t", + "GIT_AUTHOR_EMAIL": "t@example.invalid", + "GIT_COMMITTER_NAME": "t", + "GIT_COMMITTER_EMAIL": "t@example.invalid", + "LC_ALL": "C", + } + tracked = subprocess.run( + ["git", "ls-files", "-z"], cwd=REPO, check=True, capture_output=True + ).stdout.decode().split("\0") + for rel in filter(None, tracked): + src = REPO / rel + if not src.is_file(): + continue + dst = self.proj / rel + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) + _git(self.proj, "init", "-q", "-b", "main", env=self.env) + _git(self.proj, "add", "-A", env=self.env) + _git(self.proj, "commit", "-q", "-m", "skeleton", env=self.env) + for rel in self.PLANTED: + p = self.proj / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text("not part of the bootstrap\n") + + def tearDown(self) -> None: + shutil.rmtree(self.tmp) + + def _run_bootstrap(self) -> subprocess.CompletedProcess: + return subprocess.run( + ["bash", str(self.proj / "bootstrap.sh"), "--non-interactive", + "--project-name=demo", "--no-hooks"], + cwd=self.proj, env=self.env, capture_output=True, text=True, + ) + + def test_planted_files_are_neither_committed_nor_staged(self) -> None: + r = self._run_bootstrap() + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + committed = set(_git(self.proj, "ls-tree", "-r", "--name-only", "HEAD", + env=self.env).split()) + staged = set(_git(self.proj, "diff", "--cached", "--name-only", + env=self.env).split()) + for rel in self.PLANTED: + self.assertFalse(rel in committed, f"{rel} was committed by bootstrap") + self.assertFalse(rel in staged, f"{rel} was staged by bootstrap") + self.assertTrue((self.proj / rel).is_file(), f"{rel} was removed") + + def test_generated_files_are_still_committed(self) -> None: + r = self._run_bootstrap() + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + changed = set(_git(self.proj, "show", "--name-only", "--format=", "HEAD", + env=self.env).split()) + for rel in (".agentile/CONFIG.md", "CLAUDE.md", + ".agentile/coverage/baseline.json"): + self.assertIn(rel, changed) + self.assertTrue(any(p.startswith(".agentile/sprints/active/") and + "YYYY-XX-XX" not in p for p in changed), changed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflow_hygiene.py b/tests/test_workflow_hygiene.py new file mode 100644 index 0000000..d0bd6c1 --- /dev/null +++ b/tests/test_workflow_hygiene.py @@ -0,0 +1,116 @@ +"""Workflow hygiene pins that the medlow suite does not cover. + +* Every ``${{ }}`` expression inside a ``run:`` body must be one of a small + allowlist of values that cannot carry shell syntax (event name, SHAs, PR + number). Inputs, titles, bodies, branch names and step outputs must reach + the shell through ``env:``. +* Every ``uses:`` line, including the common ``- uses:`` list form, is pinned + to a full commit SHA (actions) or a fixed version (docker images), and + never to an abandoned or ``latest`` ref. +* Every claim-grade step that consumes ``$PR_NUMBER`` validates it first. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + +WORKFLOWS = Path(__file__).resolve().parents[1] / ".github" / "workflows" + +SAFE_EXPR = re.compile( + r"^\s*(github\.event_name" + r"|github\.event\.pull_request\.(base|head)\.sha" + r"|github\.event\.before|github\.sha" + r"|github\.event\.pull_request\.number)\s*$" +) + + +def run_bodies(text: str): + """Yield (lineno, line) for every line inside a run: body.""" + lines = text.splitlines() + in_run, run_indent = False, 0 + for n, line in enumerate(lines, 1): + if line.lstrip().startswith("#"): + continue + ind = len(line) - len(line.lstrip()) + if in_run: + if line.strip() and ind <= run_indent: + in_run = False + else: + yield n, line + continue + m = re.match(r"^(\s*)(- )?run:\s*(.*)$", line) + if m: + rest = m.group(3) + if rest.startswith(("|", ">")): + in_run, run_indent = True, len(m.group(1)) + (2 if m.group(2) else 0) + else: + yield n, rest + + +class RunBodyExpressions(unittest.TestCase): + def test_only_allowlisted_expressions_reach_a_shell(self) -> None: + bad = [] + for wf in sorted(WORKFLOWS.glob("*.yml")): + for n, line in run_bodies(wf.read_text(encoding="utf-8")): + for expr in re.findall(r"\$\{\{(.*?)\}\}", line): + if not SAFE_EXPR.match(expr): + bad.append(f"{wf.name}:{n}: ${{{{{expr}}}}}") + self.assertEqual(bad, [], "route these through env: instead") + + def test_detector_sees_a_dispatch_input_in_run(self) -> None: + sample = ( + " steps:\n" + " - name: x\n" + " run: |\n" + " ./grade.py --pr \"${{ github.event.inputs.pr }}\"\n" + ) + exprs = [e for _, l in run_bodies(sample) for e in re.findall(r"\$\{\{(.*?)\}\}", l)] + self.assertEqual(len(exprs), 1) + self.assertFalse(SAFE_EXPR.match(exprs[0])) + + +class ActionRefs(unittest.TestCase): + def test_every_uses_line_is_pinned(self) -> None: + bad = [] + for wf in sorted(WORKFLOWS.glob("*.yml")): + for n, line in enumerate(wf.read_text(encoding="utf-8").splitlines(), 1): + m = re.match(r"^\s*(?:- )?uses:\s*([^\s#]+)", line) + if not m: + continue + ref = m.group(1) + if ref.startswith("./"): + continue + if "returntocorp" in ref or ref.endswith((":latest", "@latest")): + bad.append(f"{wf.name}:{n}: {ref}") + elif ref.startswith("docker://"): + if not re.search(r"(:\d+\.\d+\.\d+|@sha256:[0-9a-f]{64})$", ref): + bad.append(f"{wf.name}:{n}: {ref}") + elif not re.search(r"@[0-9a-f]{40}$", ref): + bad.append(f"{wf.name}:{n}: {ref}") + self.assertEqual(bad, []) + + +class ClaimGradePrNumber(unittest.TestCase): + def test_every_step_using_pr_number_validates_it_first(self) -> None: + text = (WORKFLOWS / "claim-grade.yml").read_text(encoding="utf-8") + guard = '[[ ! "$PR_NUMBER" =~ ^[0-9]+$ ]]' + users = 0 + for step in re.split(r"\n\s*- name:", text): + if "run:" not in step: + continue + body = step.split("run:", 1)[1] + if "$PR_NUMBER" not in body: + continue + users += 1 + # The first shell use of $PR_NUMBER must be the numeric check. + self.assertEqual( + body.find("$PR_NUMBER"), body.find(guard) + len('[[ ! "'), + f"step '{step.splitlines()[0].strip()}' uses $PR_NUMBER before validating it", + ) + self.assertGreater(users, 0) + + +if __name__ == "__main__": + unittest.main()