diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 45c2778..76a2e27 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -28,5 +28,9 @@ jobs: runs-on: [self-hosted] steps: - uses: actions/checkout@v4 + - name: Checker tests + run: python3 tests/test_runner_policy.py - name: Audit this repository's workflows and exception expiry + env: + POLICY_REPO: github-policy run: bash scripts/audit-workflows.sh . diff --git a/.github/workflows/runner-policy-reusable.yml b/.github/workflows/runner-policy-reusable.yml index e23a9db..e063e1e 100644 --- a/.github/workflows/runner-policy-reusable.yml +++ b/.github/workflows/runner-policy-reusable.yml @@ -13,18 +13,16 @@ # Making this block a merge requires adding `runner-policy` as a required # status check on the calling repository's protected branch. # -# The checker is inlined deliberately. This repository is private, so a calling -# repository cannot fetch scripts/audit-workflows.sh from it: raw.github- -# usercontent.com returns 404 unauthenticated, and the caller's GITHUB_TOKEN -# has no read access here either. Because the caller resolves this workflow -# from this repository, the policy still has one source of truth -- this file. -# Keep the logic below in sync with scripts/audit-workflows.sh, the local -# checker used by audit.yml. +# The checker used to be inlined here, because a private policy repository +# cannot be checked out by a public caller. This repository is public as of +# 2026-08-17, so the caller checks it out and runs scripts/runner_policy.py +# directly. That is now the only copy of the rule -- do not re-inline it. # -# Exceptions: this gate is strict and reads no exceptions file (it cannot see -# one). An owner-approved exception means dropping the required status check on -# that repository, and recording the reason and expiry in -# runner-exceptions.json so the daily expiry check still surfaces it. +# Exceptions: this gate reads runner-exceptions.json out of the second +# checkout, so an owner-approved exception no longer means dropping the +# required status check on that repository. Entries are keyed by repo and +# workflow file, optionally narrowed to named jobs, and an expired entry fails +# the gate rather than suppressing anything. # # Labels name capabilities, not locations (2026-08-19). `node-b` was being used # as a fleet selector, which left two of the five runners permanently idle @@ -51,76 +49,20 @@ jobs: runner-policy: runs-on: [self-hosted] steps: - - uses: actions/checkout@v5 + - name: Check out the calling repository + uses: actions/checkout@v5 + + - name: Check out the policy + uses: actions/checkout@v5 + with: + repository: TheDancingDeveloper-org/github-policy + ref: main + path: .runner-policy - name: Audit runner selection shell: bash run: | set -euo pipefail - python3 - <<'PY' - import pathlib - import re - import sys - - def indent(line: str) -> int: - return len(line) - len(line.lstrip(' ')) - - def values(lines: list[str], index: int) -> list[str]: - line = lines[index] - base = indent(line) - value = line.split(':', 1)[1].split('#', 1)[0].strip() - if value: - return [p.strip().strip('"\'') for p in value.strip('[]').split(',') if p.strip()] - result = [] - for child in lines[index + 1:]: - stripped = child.strip() - if not stripped or stripped.startswith('#'): - continue - if indent(child) <= base: - break - match = re.match(r'^-\s*([^#]+)', stripped) - if match: - result.append(match.group(1).strip().strip('"\'')) - return result - - # Assembled at runtime on purpose. A literal dollar-brace-brace in - # this file would be parsed as a GitHub Actions expression before the - # script ever runs, which breaks the whole workflow -- the first - # version of this file failed with an unresolvable workflow name for - # exactly that reason. - EXPR = '$' + '{' + '{' - - failed = False - workflows = sorted( - p for pattern in ('.github/workflows/*.yml', '.github/workflows/*.yaml') - for p in pathlib.Path('.').glob(pattern) - ) - if not workflows: - print('no workflows found; nothing to audit') - - for path in workflows: - lines = path.read_text(encoding='utf-8').splitlines() - for number, line in enumerate(lines): - if not re.match(r'^\s*runs-on\s*:', line, re.I): - continue - selected = values(lines, number) - if not any(v.lower() == 'self-hosted' for v in selected): - print(f'{path}:{number + 1}: runner selection is not explicitly ' - f'self-hosted -> {selected}', file=sys.stderr) - failed = True - if any(EXPR in v for v in selected): - print(f'{path}:{number + 1}: dynamic runner selection requires ' - f'explicit review', file=sys.stderr) - failed = True - - if failed: - print('', file=sys.stderr) - print('Organization policy: every job must select a self-hosted runner ' - 'explicitly, e.g.', file=sys.stderr) - print(' runs-on: [self-hosted] # any runner', file=sys.stderr) - print(' runs-on: [self-hosted, publish] # needs Docker', file=sys.stderr) - print(' runs-on: [self-hosted, tailnet] # needs Tailnet', file=sys.stderr) - sys.exit(1) - - print(f'ok: {len(workflows)} workflow file(s) audited, all self-hosted') - PY + python3 .runner-policy/scripts/runner_policy.py . \ + --repo "${GITHUB_REPOSITORY##*/}" \ + --exceptions .runner-policy/runner-exceptions.json diff --git a/runner-exceptions.json b/runner-exceptions.json index 18a7268..4a0270e 100644 --- a/runner-exceptions.json +++ b/runner-exceptions.json @@ -1,4 +1,35 @@ { "schema_version": 1, - "exceptions": [] + "exceptions": [ + { + "repo": "cadastre", + "workflow": "ci.yaml", + "reason": "Public repository, and this repository enforces a stricter rule than the org's in its own test suite: tests/test_release_workflow.py pins SELF_HOSTED_WORKFLOWS to the two tag-gated release workflows and asserts every other job is NOT self-hosted, because pull_request builds untrusted fork code and the self-hosted pool is persistent, runs as root and sits on the tailnet. Moving ci.yaml or publish.yml would require weakening that test. Revisit only alongside ephemeral runners and fork-PR approval set to all outside contributors.", + "expires_on": "2027-02-17" + }, + { + "repo": "cadastre", + "workflow": "publish.yml", + "reason": "Same tested invariant as cadastre/ci.yaml. publish.yml triggers on push to main, so it has no fork exposure, but test_self_hosted_workflows_are_all_tag_gated additionally requires that anything self-hosted be reachable only from refs/tags/v*. The tag-gated release-images.yml and release-pypi.yml are self-hosted and stay that way.", + "expires_on": "2027-02-17" + }, + { + "repo": "FarmEggs", + "workflow": "ci.yml", + "jobs": [ + "mobile" + ], + "reason": "flutter build apk --debug needs a JDK and the Android SDK. Neither is on any self-hosted runner and subosito/flutter-action does not install them; verified absent on all three runner containers 2026-08-17. Move when the runner image ships them.", + "expires_on": "2026-11-17" + }, + { + "repo": "FarmEggs", + "workflow": "ci.yml", + "jobs": [ + "web" + ], + "reason": "The browser suite and the M1a acceptance run need google-chrome at the path CHROME_EXECUTABLE/EGGS_WEB_CHROME pin, plus npm for playwright-core. Verified absent on all three runner containers 2026-08-17. Move when the runner image ships them.", + "expires_on": "2026-11-17" + } + ] } diff --git a/scripts/audit-workflows.sh b/scripts/audit-workflows.sh index 9933446..4c5811e 100755 --- a/scripts/audit-workflows.sh +++ b/scripts/audit-workflows.sh @@ -1,67 +1,12 @@ #!/usr/bin/env bash +# +# Thin wrapper kept for the existing call sites. All logic is in +# scripts/runner_policy.py, which the org-wide gate runs directly. set -euo pipefail root="${1:-.}" -today="${POLICY_DATE:-$(date -u +%F)}" -exceptions="${RUNNER_EXCEPTION_FILE:-$root/runner-exceptions.json}" -failed=0 - -command -v jq >/dev/null -jq -e '.schema_version == 1 and (.exceptions | type == "array")' "$exceptions" >/dev/null - -while IFS= read -r expiry; do - [[ "$expiry" > "$today" || "$expiry" == "$today" ]] || { - printf 'expired runner exception: %s\n' "$expiry" >&2 - failed=1 - } -done < <(jq -r '.exceptions[].expires_on' "$exceptions") - -while IFS= read -r -d '' workflow; do - if ! python3 - "$workflow" <<'PY' -import re -import sys - -path = sys.argv[1] -lines = open(path, encoding='utf-8').read().splitlines() - -def indent(line: str) -> int: - return len(line) - len(line.lstrip(' ')) - -def values(index: int) -> list[str]: - line = lines[index] - base = indent(line) - value = line.split(':', 1)[1].split('#', 1)[0].strip() - if value: - return [part.strip().strip('"\'') for part in value.strip('[]').split(',') if part.strip()] - result = [] - for child in lines[index + 1:]: - stripped = child.strip() - if not stripped or stripped.startswith('#'): - continue - if indent(child) <= base: - break - match = re.match(r'^-\s*([^#]+)', stripped) - if match: - result.append(match.group(1).strip().strip('"\'')) - return result - -failed = False -for number, line in enumerate(lines): - if not re.match(r'^\s*runs-on\s*:', line, re.I): - continue - selected = values(number) - if not any(value.lower() == 'self-hosted' for value in selected): - print(f'{path}:{number + 1}: runner selection is not explicitly self-hosted', file=sys.stderr) - failed = True - if any('${{' in value for value in selected): - print(f'{path}:{number + 1}: dynamic runner selection requires explicit review', file=sys.stderr) - failed = True -sys.exit(1 if failed else 0) -PY - then - failed=1 - fi -done < <(find "$root" -path '*/.git' -prune -o -path '*/.github/workflows/*.yml' -print0 -o -path '*/.github/workflows/*.yaml' -print0) - -exit "$failed" +exec python3 "$(dirname "$0")/runner_policy.py" "$root" \ + ${POLICY_REPO:+--repo "$POLICY_REPO"} \ + --exceptions "${RUNNER_EXCEPTION_FILE:-$root/runner-exceptions.json}" \ + ${POLICY_DATE:+--today "$POLICY_DATE"} diff --git a/scripts/runner_policy.py b/scripts/runner_policy.py new file mode 100755 index 0000000..34898df --- /dev/null +++ b/scripts/runner_policy.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Audit `runs-on:` selections in a repository's workflows. + +The single source of truth for the rule. Both entry points use this file: + + * `.github/workflows/audit.yml` -> `scripts/audit-workflows.sh` (this repo) + * `.github/workflows/runner-policy-reusable.yml` (every calling repository) + +The reusable gate used to carry an inlined copy of this logic, because a +private policy repository cannot be checked out by a public caller. This +repository is public now, so the caller checks it out and runs this file +directly. Do not re-inline it -- two copies of a policy drift. + +Deliberately dependency-free and line-based rather than YAML-parsing: it runs +on the self-hosted runner image, which ships no PyYAML. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import date +from pathlib import Path + +# A literal dollar-brace-brace anywhere in a workflow file would be parsed as a +# GitHub Actions expression before the job ever runs. This module is a separate +# file rather than a heredoc inside a workflow, so it is safe here -- but the +# constant stays for the benefit of anyone tempted to inline it again. +EXPR = "${{" + + +def indent(line: str) -> int: + return len(line) - len(line.lstrip(" ")) + + +def values(lines: list[str], index: int) -> list[str]: + """The runner labels selected by the `runs-on:` at `index`. + + Handles both the inline form (`runs-on: [a, b]`, `runs-on: a`) and the + block-sequence form spread over following lines. + """ + line = lines[index] + base = indent(line) + value = line.split(":", 1)[1].split("#", 1)[0].strip() + if value: + return [p.strip().strip("\"'") for p in value.strip("[]").split(",") if p.strip()] + result: list[str] = [] + for child in lines[index + 1:]: + stripped = child.strip() + if not stripped or stripped.startswith("#"): + continue + if indent(child) <= base: + break + match = re.match(r"^-\s*([^#]+)", stripped) + if match: + result.append(match.group(1).strip().strip("\"'")) + return result + + +def job_at(lines: list[str], index: int) -> str | None: + """The job key enclosing the line at `index`. + + Exceptions are scoped per job, not per file: FarmEggs' ci.yml holds two + compliant jobs and two excepted ones, and a file-wide exception there would + also hide a future regression in the compliant pair. + """ + jobs_indent = None + for number in range(index, -1, -1): + if re.match(r"^jobs\s*:", lines[number]): + jobs_indent = indent(lines[number]) + break + if jobs_indent is None: + return None + want = jobs_indent + 2 + for number in range(index, -1, -1): + line = lines[number] + if not line.strip() or line.strip().startswith("#"): + continue + if indent(line) != want: + continue + match = re.match(r"^\s*([A-Za-z_][A-Za-z0-9_.-]*)\s*:\s*(#.*)?$", line) + if match: + return match.group(1) + return None + + +def load_exceptions(path: Path, today: str) -> tuple[list[dict], list[str]]: + """Return (live exceptions, expiry errors). + + An expired entry is a hard failure in its own right -- that is what stops + exceptions rotting silently -- and it never suppresses anything. + """ + if not path.exists(): + return [], [] + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("schema_version") != 1 or not isinstance(payload.get("exceptions"), list): + return [], [f"{path}: not a schema_version 1 exceptions file"] + + live: list[dict] = [] + errors: list[str] = [] + for entry in payload["exceptions"]: + expiry = str(entry.get("expires_on", "")) + if not re.match(r"^\d{4}-\d{2}-\d{2}$", expiry): + errors.append(f"{path}: exception for {entry.get('repo')}/{entry.get('workflow')} " + f"has no valid expires_on") + continue + if expiry < today: + errors.append(f"expired runner exception: {entry.get('repo')}/" + f"{entry.get('workflow')} expired {expiry}") + continue + live.append(entry) + return live, errors + + +def excused(exceptions: list[dict], repo: str, workflow: str, job: str | None) -> dict | None: + for entry in exceptions: + if entry.get("repo") != repo or entry.get("workflow") != workflow: + continue + scope = entry.get("jobs") + if scope is None or (job is not None and job in scope): + return entry + return None + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("root", nargs="?", default=".", + help="repository checkout to audit") + parser.add_argument("--repo", default="", + help="repository name the exceptions file is keyed by") + parser.add_argument("--exceptions", default="", + help="path to runner-exceptions.json") + parser.add_argument("--today", default=date.today().isoformat()) + args = parser.parse_args() + + root = Path(args.root) + repo = args.repo or root.resolve().name + exceptions_path = Path(args.exceptions) if args.exceptions else root / "runner-exceptions.json" + exceptions, errors = load_exceptions(exceptions_path, args.today) + for error in errors: + print(error, file=sys.stderr) + failed = bool(errors) + + workflows = sorted( + p for pattern in (".github/workflows/*.yml", ".github/workflows/*.yaml") + for p in root.glob(pattern) + ) + if not workflows: + print("no workflows found; nothing to audit") + + excused_count = 0 + for path in workflows: + lines = path.read_text(encoding="utf-8").splitlines() + for number, line in enumerate(lines): + if not re.match(r"^\s*runs-on\s*:", line, re.I): + continue + selected = values(lines, number) + hosted = not any(v.lower() == "self-hosted" for v in selected) + dynamic = any(EXPR in v for v in selected) + if not hosted and not dynamic: + continue + + job = job_at(lines, number) + entry = excused(exceptions, repo, path.name, job) + if entry is not None: + excused_count += 1 + print(f"{path}:{number + 1}: excused until {entry['expires_on']} " + f"({entry.get('reason', 'no reason recorded')})") + continue + + where = f"{path}:{number + 1}" + if job: + where += f" (job {job})" + if hosted: + print(f"{where}: runner selection is not explicitly self-hosted " + f"-> {selected}", file=sys.stderr) + if dynamic: + print(f"{where}: dynamic runner selection requires explicit review", + file=sys.stderr) + failed = True + + if failed: + print("", file=sys.stderr) + print("Organization policy: every job must select a self-hosted runner " + "explicitly, e.g.", file=sys.stderr) + print(" runs-on: [self-hosted, node-b, linux, x64]", file=sys.stderr) + print(" runs-on: [self-hosted, node-b, linux, x64, docker, publish] " + "# needs Docker", file=sys.stderr) + print("", file=sys.stderr) + print("An owner-approved exception goes in github-policy/" + "runner-exceptions.json.", file=sys.stderr) + return 1 + + suffix = f", {excused_count} excused" if excused_count else "" + print(f"ok: {len(workflows)} workflow file(s) audited, all self-hosted{suffix}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_runner_policy.py b/tests/test_runner_policy.py new file mode 100644 index 0000000..b313345 --- /dev/null +++ b/tests/test_runner_policy.py @@ -0,0 +1,195 @@ +"""Tests for the runner-policy checker. + +Run with `python3 -m pytest tests/` or plain `python3 tests/test_runner_policy.py`. +No third-party imports, so it runs on the self-hosted image as-is. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +CHECKER = Path(__file__).resolve().parent.parent / "scripts" / "runner_policy.py" +TODAY = "2026-08-17" + + +def audit(workflows: dict[str, str], exceptions: dict | None = None, + repo: str = "example", today: str = TODAY) -> tuple[int, str]: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / ".github" / "workflows").mkdir(parents=True) + for name, body in workflows.items(): + (root / ".github" / "workflows" / name).write_text(body) + exceptions_path = root / "runner-exceptions.json" + exceptions_path.write_text(json.dumps(exceptions or {"schema_version": 1, "exceptions": []})) + result = subprocess.run( + [sys.executable, str(CHECKER), str(root), "--repo", repo, + "--exceptions", str(exceptions_path), "--today", today], + capture_output=True, text=True, + ) + return result.returncode, result.stdout + result.stderr + + +COMPLIANT = """name: CI +jobs: + build: + runs-on: [self-hosted, node-b, linux, x64] + steps: + - run: true +""" + +HOSTED = """name: CI +jobs: + build: + runs-on: ubuntu-latest + steps: + - run: true +""" + +TWO_JOBS = """name: CI +jobs: + good: + runs-on: [self-hosted, node-b, linux, x64] + steps: + - run: true + bad: + runs-on: ubuntu-latest + steps: + - run: true +""" + + +def test_compliant_passes(): + code, out = audit({"ci.yml": COMPLIANT}) + assert code == 0, out + + +def test_hosted_fails(): + code, out = audit({"ci.yml": HOSTED}) + assert code == 1 + assert "not explicitly self-hosted" in out + + +def test_dynamic_selection_fails(): + code, out = audit({"ci.yml": HOSTED.replace("ubuntu-latest", "${{ matrix.os }}")}) + assert code == 1 + assert "dynamic runner selection" in out + + +def test_block_sequence_form_is_understood(): + code, out = audit({"ci.yml": """name: CI +jobs: + build: + runs-on: + - self-hosted + - node-b + steps: + - run: true +"""}) + assert code == 0, out + + +def test_commented_out_runs_on_is_not_a_violation(): + # Several repos carry `# runs-on: ubuntu-latest` in a warning comment. + code, out = audit({"ci.yml": COMPLIANT + " # `runs-on: ubuntu-latest` is rejected here\n"}) + assert code == 0, out + + +def test_file_wide_exception_excuses(): + code, out = audit( + {"ci.yml": HOSTED}, + {"schema_version": 1, "exceptions": [ + {"repo": "example", "workflow": "ci.yml", "reason": "r", "expires_on": "2026-12-01"}]}, + ) + assert code == 0, out + assert "excused" in out + + +def test_job_scoped_exception_does_not_excuse_a_sibling(): + code, out = audit( + {"ci.yml": TWO_JOBS.replace(""" good: + runs-on: [self-hosted, node-b, linux, x64]""", """ good: + runs-on: ubuntu-latest""")}, + {"schema_version": 1, "exceptions": [ + {"repo": "example", "workflow": "ci.yml", "jobs": ["bad"], + "reason": "r", "expires_on": "2026-12-01"}]}, + ) + assert code == 1, out + assert "job good" in out + assert "job bad" not in out + + +def test_job_scoped_exception_excuses_its_own_job(): + code, out = audit( + {"ci.yml": TWO_JOBS}, + {"schema_version": 1, "exceptions": [ + {"repo": "example", "workflow": "ci.yml", "jobs": ["bad"], + "reason": "r", "expires_on": "2026-12-01"}]}, + ) + assert code == 0, out + + +def test_exception_for_another_repo_does_not_apply(): + code, out = audit( + {"ci.yml": HOSTED}, + {"schema_version": 1, "exceptions": [ + {"repo": "somewhere-else", "workflow": "ci.yml", "reason": "r", + "expires_on": "2026-12-01"}]}, + ) + assert code == 1, out + + +def test_expired_exception_fails_and_suppresses_nothing(): + code, out = audit( + {"ci.yml": HOSTED}, + {"schema_version": 1, "exceptions": [ + {"repo": "example", "workflow": "ci.yml", "reason": "r", "expires_on": "2026-08-16"}]}, + ) + assert code == 1 + assert "expired runner exception" in out + assert "not explicitly self-hosted" in out + + +def test_exception_expiring_today_is_still_live(): + code, out = audit( + {"ci.yml": HOSTED}, + {"schema_version": 1, "exceptions": [ + {"repo": "example", "workflow": "ci.yml", "reason": "r", "expires_on": TODAY}]}, + ) + assert code == 0, out + + +def test_exception_without_expiry_is_rejected(): + code, out = audit( + {"ci.yml": COMPLIANT}, + {"schema_version": 1, "exceptions": [{"repo": "example", "workflow": "ci.yml"}]}, + ) + assert code == 1 + assert "no valid expires_on" in out + + +def test_shipped_exceptions_file_is_valid_and_live(): + path = Path(__file__).resolve().parent.parent / "runner-exceptions.json" + payload = json.loads(path.read_text()) + assert payload["schema_version"] == 1 + for entry in payload["exceptions"]: + assert entry["repo"] and entry["workflow"] + assert len(entry["reason"]) > 40, f"{entry['repo']}: record a real reason" + assert entry["expires_on"] >= TODAY, f"{entry['repo']}: exception has expired" + + +if __name__ == "__main__": + failures = 0 + for name, fn in sorted(globals().items()): + if not name.startswith("test_") or not callable(fn): + continue + try: + fn() + print(f"ok {name}") + except AssertionError as error: + failures += 1 + print(f"FAIL {name}: {error}") + sys.exit(1 if failures else 0)