From 9028975aadfafc8d4c6b1dd91e725e1979269ca3 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Mon, 2 Nov 2026 13:41:15 +0000 Subject: [PATCH] Exempt fork parent history from the range gate A fork that merges its parent repository inherits commits whose identities, file sizes and contents it cannot change. In a GitHub fork the workflow now fetches the parent's branches into refs/upstream/* and the script excludes commits reachable from them (--upstream-ref). Identities and secrets are checked on the fork's own commits only. A (path, blob) pair introduced by a parent commit stays exempt where it reappears, for example in the fork's merge commit; any fork change to such a path, and any new path, is checked as before. On itpplasma/GPEC#10 (develop merged with PrincetonUniversity/GPEC develop) the gate previously failed on 57.6 MB of parent blobs and three parent identities; with the parent refs only the merge commit is checked, and it passes. The job timeout rises to 10 min for the parent fetch. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2kuiWu67fzfh8w4Fkby3P --- .github/workflows/repo-hygiene.yml | 20 +++++++++++++-- REPO_HYGIENE.md | 8 ++++++ scripts/repo_hygiene.py | 41 +++++++++++++++++++++++------- tests/test_repo_hygiene.py | 39 ++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index c7c9341..e67b926 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -14,7 +14,7 @@ jobs: github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.actor != 'github-merge-queue[bot]') name: repo-hygiene runs-on: ${{ github.event.pull_request.head.repo.fork && fromJSON('["self-hosted","itp-ubuntu24"]') || fromJSON('["self-hosted", "itp-light"]') }} - timeout-minutes: 5 + timeout-minutes: 10 steps: # Public consumers need Git objects, never recursive submodule/auth setup. - name: Fetch the public consumer at the exact tested revision @@ -31,6 +31,18 @@ jobs: fi git fetch --no-tags origin "${refs[@]}" GIT_LFS_SKIP_SMUDGE=1 git -c submodule.recurse=false checkout --detach "$HEAD" + # A fork's parent history is published and outside the fork's control. + - name: Fetch the parent repository of a fork + env: + REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ github.token }} + run: | + parent=$(curl -fsSL -H "Authorization: Bearer $GITHUB_TOKEN" \ + -H 'Accept: application/vnd.github+json' "https://api.github.com/repos/$REPOSITORY" | + python3 -c 'import json, sys; print((json.load(sys.stdin).get("parent") or {}).get("full_name", ""))') + if [ -n "$parent" ]; then + git fetch --no-tags "https://github.com/$parent.git" '+refs/heads/*:refs/upstream/*' + fi - uses: actions/checkout@v6 with: repository: itpplasma/.github @@ -57,4 +69,8 @@ jobs: if [ -z "$BASE" ] || [ "$BASE" = 0000000000000000000000000000000000000000 ]; then BASE=$(git rev-parse "$HEAD^") fi - python3 .hygiene-tools/scripts/repo_hygiene.py --base "$BASE" --head "$HEAD" --github-repository "${{ github.repository }}" + upstream=() + if git for-each-ref --count=1 --format=x refs/upstream/ | grep -q x; then + upstream=(--upstream-ref 'refs/upstream/*') + fi + python3 .hygiene-tools/scripts/repo_hygiene.py --base "$BASE" --head "$HEAD" --github-repository "${{ github.repository }}" "${upstream[@]}" diff --git a/REPO_HYGIENE.md b/REPO_HYGIENE.md index 1b53294..82d7ed8 100644 --- a/REPO_HYGIENE.md +++ b/REPO_HYGIENE.md @@ -39,6 +39,14 @@ add `--github-repository OWNER/REPO` with `GITHUB_TOKEN` to verify identities to The staged local check stays offline and only warns when the effective author or committer email differs from `git config user.email`; it does not block that case. +In a GitHub fork, CI fetches the parent repository's branches into +`refs/upstream/*` and passes `--upstream-ref 'refs/upstream/*'`. Commits +reachable from those refs are published history the fork does not control: +their identities, blobs and secrets are not checked, and a path stays exempt +while it carries the parent's exact blob (for example after merging the +parent). The fork's own commits, including the merge commit and its conflict +resolutions, are checked in full. + Each added/modified file blob must be at most 1 MiB. Data/archive/document types require an explicit fixture allowance. Unique, non-exempt added blob bytes sum to at most 5 MiB per PR (or staged change); repeated copies of the same blob diff --git a/scripts/repo_hygiene.py b/scripts/repo_hygiene.py index a04bbb2..1c8693b 100755 --- a/scripts/repo_hygiene.py +++ b/scripts/repo_hygiene.py @@ -137,7 +137,20 @@ def entries(raw): yield os.fsdecode(fields[i + 1]), oid -def check(staged, base, head, repository=None): +def commit_blobs(commit): + return entries(git('diff-tree', '--root', '-m', '--no-commit-id', '-r', '--raw', '-z', + '--no-abbrev', '--no-renames', '--diff-filter=AMT', commit)) + + +def upstream_exclusion(upstream): + """rev-list arguments that drop commits reachable from the fork parent's refs.""" + for pattern in upstream: + if not re.fullmatch(r'refs/[A-Za-z0-9_./*-]+', pattern) or '..' in pattern: + raise ValueError(f'invalid upstream ref pattern {pattern!r}') + return ['--not', *(f'--glob={pattern}' for pattern in upstream)] if upstream else [] + + +def check(staged, base, head, repository=None, upstream=()): commits = [] if staged: warn_local_identity() @@ -145,14 +158,20 @@ def check(staged, base, head, repository=None): '--no-renames', '--diff-filter=AMT'))) else: blobs = [] - commits = git('rev-list', '--reverse', f'{base}..{head}').decode().splitlines() + exclusion = upstream_exclusion(upstream) + commits = git('rev-list', '--reverse', f'{base}..{head}', + *exclusion).decode().splitlines() + # Commits of the fork parent are published upstream history that the + # fork does not control; a path keeps that exemption while it carries + # the parent's exact blob, for example through a merge of the parent. + inherited = set(git('rev-list', f'{base}..{head}').decode().split()) - set(commits) + published = {entry for commit in inherited for entry in commit_blobs(commit)} for commit in commits: - blobs.extend(entries(git('diff-tree', '--root', '-m', '--no-commit-id', '-r', - '--raw', '-z', '--no-abbrev', '--no-renames', - '--diff-filter=AMT', commit))) + blobs.extend(commit_blobs(commit)) # Also inspect the net range diff; -m above checks each merge parent. blobs.extend(entries(git('diff', '--raw', '-z', '--no-abbrev', '--no-renames', '--diff-filter=AMT', base, head))) + blobs = [entry for entry in blobs if entry not in published] rules, allowed_identities = allowances(None if staged else head) if staged: rules = rules + local_allowances() @@ -193,7 +212,8 @@ def check(staged, base, head, repository=None): args = ['gitleaks', 'protect' if staged else 'detect', '--no-banner', '--redact', '--log-level=error', '--config', str(config), '--gitleaks-ignore-path', os.devnull, '--ignore-gitleaks-allow'] - args += ['--staged'] if staged else ['--log-opts', f'{base}..{head}'] + args += ['--staged'] if staged else [ + '--log-opts', ' '.join([f'{base}..{head}', *upstream_exclusion(upstream)])] env = {k: v for k, v in os.environ.items() if not k.startswith('GITLEAKS_')} return subprocess.run(args, env=env).returncode @@ -204,14 +224,17 @@ def main(): parser.add_argument('--base') parser.add_argument('--head', default='HEAD') parser.add_argument('--github-repository', help='verify range identities via GitHub (needs GITHUB_TOKEN)') + parser.add_argument('--upstream-ref', action='append', default=[], metavar='GLOB', + help='exempt range commits reachable from these refs (fork parent)') args = parser.parse_args() if args.staged == bool(args.base): parser.error('choose --staged or --base REF [--head REF]') - if args.staged and args.github_repository: - parser.error('--github-repository requires --base') + if args.staged and (args.github_repository or args.upstream_ref): + parser.error('--github-repository and --upstream-ref require --base') os.chdir(git('rev-parse', '--show-toplevel').decode().strip()) try: - return check(args.staged, args.base, args.head, args.github_repository) + return check(args.staged, args.base, args.head, args.github_repository, + args.upstream_ref) except (OSError, ValueError, subprocess.CalledProcessError) as exc: print(f'repo-hygiene: {exc}', file=sys.stderr) return 2 diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py index 50d013e..21e8761 100644 --- a/tests/test_repo_hygiene.py +++ b/tests/test_repo_hygiene.py @@ -93,6 +93,45 @@ def test_secret_staged_and_range(self): self.git('commit', '-qm', 'remove') self.assertEqual(self.gate('--base', self.base).returncode, 1) + def fork_merge(self): + """Merge a parent-repository branch whose history the fork does not control.""" + self.git('checkout', '-qb', 'parent') + self.add('docs/manual.pdf') + self.add('example/eq.geqdsk', b'y' * (1024 * 1024 + 1)) + for i in range(6): + self.add(f'src/{i}.f', bytes([65 + i]) * (900 * 1024)) + token = 'gh' + 'p_' + 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8' + self.add('credentials.txt', ('token=' + token + '\n').encode()) + self.git('commit', '-qm', 'parent history') + self.git('update-ref', 'refs/upstream/main', 'HEAD') + self.git('checkout', '-q', '-') + self.add('own.txt') + self.git('commit', '-qm', 'fork change') + self.git('merge', '--no-ff', '-qm', 'merge parent', 'parent') + + def test_fork_parent_history_is_exempt(self): + self.fork_merge() + self.assertEqual(self.gate('--base', self.base).returncode, 1) + result = self.gate('--base', self.base, '--upstream-ref', 'refs/upstream/*') + self.assertEqual(result.returncode, 0, result.stderr) + + def test_fork_changes_after_parent_merge_are_checked(self): + self.fork_merge() + self.add('example/eq.geqdsk', b'z' * (1024 * 1024 + 1)) + self.git('commit', '-qm', 'fork edits a parent file') + result = self.gate('--base', self.base, '--upstream-ref', 'refs/upstream/*') + self.assertIn('example/eq.geqdsk', result.stderr) + self.git('reset', '-q', '--hard', 'HEAD^') + self.add('trace.npy') + self.git('commit', '-qm', 'fork output') + result = self.gate('--base', self.base, '--upstream-ref', 'refs/upstream/*') + self.assertIn('trace.npy', result.stderr) + self.assertNotIn('manual.pdf', result.stderr) + + def test_upstream_ref_pattern_and_mode(self): + self.assertEqual(self.gate('--base', self.base, '--upstream-ref', 'HEAD').returncode, 2) + self.assertEqual(self.gate('--staged', '--upstream-ref', 'refs/upstream/*').returncode, 2) + def local_allow(self): common = Path(self.git('rev-parse', '--git-common-dir').strip()) if not common.is_absolute():