Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion REPO_HYGIENE.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ to at most 5 MiB per PR (or staged change); repeated copies of the same blob
count once. Modified files count at their full new size. Historic, unchanged
files are not charged. A rename is inspected as an addition.

Exceptions come only from the committed head (or staged index) file
Shared exceptions come from the committed head (or staged index) file
`.repo-hygiene-allow`: repository-relative glob, TAB, one-line reason. Blank
lines and `#` comments are accepted. Review allowances as part of the PR.
For example, `tests/fixtures/**` followed by a TAB and
Expand All @@ -54,5 +54,12 @@ Only synthetic or published data belong in fixtures. Never commit non-public
machine/experiment or collaborator data (ITER/TC24/IMAS, AUG shots, JT-60SA).
Keep campaign outputs outside Git, with checksums and a recorded location.

For staged checks in external repositories, keep local path exceptions in
`$(git rev-parse --git-common-dir)/info/repo-hygiene-allow`. This file is shared
by the repository’s worktrees and stays outside commits. Use `glob<TAB>reason`,
as in `.repo-hygiene-allow`; absolute patterns, missing reasons and `identities:`
sections are rejected. Range and CI checks ignore this file, and its rules never
waive secret detection.

Heavy CI runs only in public repositories. Private repositories validate via
cluster jobs recorded in the PR; GitHub Free cannot enforce private rulesets.
53 changes: 44 additions & 9 deletions scripts/repo_hygiene.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
#!/usr/bin/env python3
"""Staged / commit-range gate. Exceptions live in .repo-hygiene-allow."""
"""Staged / commit-range gate. Exceptions live in .repo-hygiene-allow.

Only --staged local checks additionally read untracked path exceptions from
$(git rev-parse --git-common-dir)/info/repo-hygiene-allow so external
repositories can keep local artifacts without committing owner tooling.
"""
import argparse
import fnmatch
import json
Expand All @@ -21,34 +26,57 @@ def git(*args, **kwargs):
return subprocess.check_output(['git', *args], **kwargs)


def allowances(ref):
name = f'{ref}:.repo-hygiene-allow' if ref else ':.repo-hygiene-allow'
if subprocess.run(['git', 'cat-file', '-e', name], stderr=subprocess.DEVNULL).returncode:
return [], set()
text = git('show', name).decode('utf-8')
def parse_allow(text, source, identities_allowed=True):
rules, identities = [], set()
section = "paths"
for number, line in enumerate(text.splitlines(), 1):
if not line.strip() or line.lstrip().startswith('#'):
continue
if line.strip() in ('identities:', 'paths:'):
section = line.strip()[:-1]
if section == 'identities' and not identities_allowed:
raise ValueError(f'{source}:{number}: local allowances cannot exempt '
'identities; only path exceptions are permitted')
continue
fields = line.split('\t', 1)
if len(fields) != 2 or not all(field.strip() for field in fields):
raise ValueError(f'.repo-hygiene-allow:{number}: expected {section} entry<TAB>reason')
raise ValueError(f'{source}:{number}: expected {section} entry<TAB>reason')
pattern, reason = (field.strip() for field in fields)
if section == 'identities':
if not re.fullmatch(r'[^\s<>@*?]+@[^\s<>@*?]+', pattern):
raise ValueError(f'.repo-hygiene-allow:{number}: expected exact email<TAB>reason')
raise ValueError(f'{source}:{number}: expected exact email<TAB>reason')
identities.add(pattern.casefold())
continue
if pattern.startswith('/') or '..' in pattern.split('/'):
raise ValueError(f'.repo-hygiene-allow:{number}: use repository-relative globs')
raise ValueError(f'{source}:{number}: use repository-relative globs')
rules.append((pattern, reason))
return rules, identities


def allowances(ref):
name = f'{ref}:.repo-hygiene-allow' if ref else ':.repo-hygiene-allow'
if subprocess.run(['git', 'cat-file', '-e', name], stderr=subprocess.DEVNULL).returncode:
return [], set()
return parse_allow(git('show', name).decode('utf-8'), '.repo-hygiene-allow')


def local_allow_file():
common = Path(git('rev-parse', '--git-common-dir').decode().strip())
return (common if common.is_absolute() else Path.cwd() / common) / 'info' / 'repo-hygiene-allow'


def local_allowances():
"""Path exceptions outside tracked content, for --staged checks only.

Never grants identity exemptions; secrets are never waived because
gitleaks runs after path allowances regardless of these rules."""
path = local_allow_file()
if not path.exists():
return []
rules, _ = parse_allow(path.read_text('utf-8'), str(path), identities_allowed=False)
return rules


def warn_local_identity():
"""Offline and advisory: Git environment overrides can change either identity."""
configured = subprocess.run(['git', 'config', 'user.email'], capture_output=True,
Expand Down Expand Up @@ -126,6 +154,8 @@ def check(staged, base, head, repository=None):
blobs.extend(entries(git('diff', '--raw', '-z', '--no-abbrev', '--no-renames',
'--diff-filter=AMT', base, head)))
rules, allowed_identities = allowances(None if staged else head)
if staged:
rules = rules + local_allowances()
oids = sorted({oid for _, oid in blobs})
sizes = {}
if oids:
Expand All @@ -149,6 +179,11 @@ def check(staged, base, head, repository=None):
print('\n'.join(failures[:30]), file=sys.stderr)
print('Archive results externally; justify synthetic/published fixtures in '
'.repo-hygiene-allow (glob<TAB>reason). No environment override.', file=sys.stderr)
if staged:
print(f'In an external repository add the reasoned exception to the local file '
f'{local_allow_file()} instead of committing owner tooling there; '
'local rules cover paths only and never waive secret detection.',
file=sys.stderr)
return 1
# Force upstream defaults: repo config, ignore files, inline comments and env
# must not silently bypass secret detection. Path allowances never waive secrets.
Expand Down
56 changes: 56 additions & 0 deletions tests/test_repo_hygiene.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,12 +93,68 @@ def test_secret_staged_and_range(self):
self.git('commit', '-qm', 'remove')
self.assertEqual(self.gate('--base', self.base).returncode, 1)

def local_allow(self):
common = Path(self.git('rev-parse', '--git-common-dir').strip())
if not common.is_absolute():
common = self.repo / common
info = common / 'info'
info.mkdir(parents=True, exist_ok=True)
return info / 'repo-hygiene-allow'

def test_staged_local_allowance(self):
self.add('runs/output.npz')
result = self.gate()
self.assertEqual(result.returncode, 1)
self.assertIn(str(self.local_allow()), result.stderr)
allow = self.local_allow()
allow.write_text('runs/**\tLocal scratch artifact from run.py\n')
self.assertEqual(self.gate().returncode, 0) # passes with reasoned local rule
self.assertNotIn('repo-hygiene-allow', self.git('status', '--porcelain'))
allow.write_text('runs/**\n') # missing reason
self.assertEqual(self.gate().returncode, 2) # malformed fails
allow.unlink()
self.assertEqual(self.gate().returncode, 1) # missing file: blocked again

def test_local_allowance_never_waives_secrets(self):
token = 'gh' + 'p_' + 'A1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q7r8'
self.add('runs/notes.txt', ('token=' + token + '\n').encode())
self.local_allow().write_text('runs/**\tLocal scratch output\n')
result = self.gate()
self.assertEqual(result.returncode, 1) # gitleaks still runs and blocks

def test_local_allowance_rejects_identities_and_absolute_paths(self):
self.add('runs/output.npz')
self.local_allow().write_text('identities:\nbot@example.org\tSome bot\n')
self.assertEqual(self.gate().returncode, 2)
self.local_allow().write_text('/runs/**\tAbsolute pattern\n')
self.assertEqual(self.gate().returncode, 2)

def test_range_ignores_local_allowance(self):
self.local_allow().write_text('trace.npy\tLocal scratch artifact\n')
self.add('trace.npy')
self.git('commit', '-qm', 'add output')
result = self.gate('--base', self.base)
self.assertEqual(result.returncode, 1)
self.assertIn('.repo-hygiene-allow (glob<TAB>reason)', result.stderr)
self.assertNotIn(str(self.local_allow()), result.stderr)

def test_tracked_allowance_applies_to_range(self):
(self.repo / '.repo-hygiene-allow').write_text(
'trace.npy\tSynthetic fixture from generator.py\n')
self.git('add', '.repo-hygiene-allow')
self.git('commit', '-qm', 'allow fixture')
self.add('trace.npy')
self.git('commit', '-qm', 'add fixture')
self.assertEqual(self.gate('--base', self.base).returncode, 0)

def test_linked_worktree(self):
worktree = self.repo.parent / 'worktree'
self.git('worktree', 'add', '-qb', 'worktree-test', str(worktree))
self.repo = worktree
self.add('results/archive.csv.gz')
self.assertEqual(self.gate().returncode, 1)
self.local_allow().write_text('results/**\tLocal scratch output\n')
self.assertEqual(self.gate().returncode, 0)


if __name__ == '__main__':
Expand Down